Merge branch 'devel' into fix_15097

Finishes `genEnumInfo` as followup to #24351. As #24381 mentions this
covers every use of `for` loops in the codegen.

---------

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -7,17 +7,16 @@ body:
 - type: markdown
   attributes:
     value: |
-      - **Please provide a minimal code example that reproduces the Bug!** :bug:
-        Reports with a reproducible example and descriptive detailed information will likely receive fixes faster.
+      Please provide a minimal code example that reproduces the bug if possible.
+      Reports with a reproducible example or detailed information will likely receive fixes faster.
 
 - type: textarea
   id: description
   attributes:
     label: Description
     description: |
-      Use DETAILED DESCRIPTIVE information about the problem.
-      Here, you go into more details about your Bug report. This section can be a few paragraphs long.
-    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
+      Describe the problem. Code example can be given here.
+    placeholder: Bug reports with reproducible code or detailed information will be fixed faster.
   validations:
     required: true
     
@@ -25,7 +24,9 @@ body:
   id: nim-version
   attributes:
     label: Nim Version
-    description: Copy and paste the output of `nim -v` on the command line. For development versions, make sure to include the commit hash.
+    description: |
+      Can be obtained from `nim -v` on the command line along with the OS/architecture.
+      For development versions, make sure to include the commit hash.
   validations:
     required: true
 
@@ -34,7 +35,7 @@ body:
   attributes:
     label: Current Output
     description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
+    placeholder: Bug reports with reproducible code or detailed information will be fixed faster.
     render: text
 
 - type: textarea
@@ -42,14 +43,14 @@ body:
   attributes:
     label: Expected Output
     description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
+    placeholder: Bug reports with reproducible code or detailed information will be fixed faster.
     render: text
 
 - type: textarea
-  id: possible-solution
+  id: known-workarounds
   attributes:
-    label: Possible Solution
-    description: Propose a possible solution.
+    label: Known Workarounds
+    description: Provide any known workarounds.
   validations:
     required: false
 
@@ -64,13 +65,9 @@ body:
 - type: markdown
   attributes:
     value: |
-      - Thanks for your contributions!, your Bug report will receive feedback from the community soon...
-      - Please check whether the problem still exists in the devel branch, see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
+      - Please check whether the problem still exists in the devel branch, which can be installed via [choosenim](https://github.com/nim-lang/choosenim/), [nightlies](https://github.com/nim-lang/nightlies/), or by [creating a temporary build of the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler-building-an-instrumented-compiler).
       - Consider writing a PR targetting devel branch after filing this, see [contributing](https://nim-lang.github.io/Nim/contributing.html).
-      - If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
-        which should give more context on a compiler crash.
-      - If it's a regression, you can help us by identifying which version introduced the bug,
-        see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
-        or at least try known past releases (e.g. `choosenim 2.0.0`). The Nim repo also supports online bisecting
-        via making a comment, which contains a code block starting by `!nim c`, `!nim js` etc. , see [nimrun-action](https://github.com/juancarlospaco/nimrun-action).
+      - If it's a pre-existing compiler bug, see [debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler) which should give more context on a compiler crash.
+      - If it's a regression, you can help us by identifying which version introduced the bug by [bisecting](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions) or at least trying known past releases (e.g. `choosenim 2.0.0`).
+        The Nim repo also supports bisecting in issue comments by adding `!nim c`, `!nim js` etc. before a code block, see [nimrun-action](https://github.com/juancarlospaco/nimrun-action).
       - [Please, consider a Donation for the Nim project.](https://nim-lang.org/donate.html)

.github/workflows/ci_bench.yml

@@ -1,115 +0,0 @@
-name: Benchmarks CI
-on:
-  pull_request:
-  push:
-    branches:
-      - 'devel'
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-20.04]
-        cpu: [amd64]
-    name: '${{ matrix.os }}'
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 60 # refs bug #18178
-    steps:
-      - name: 'Checkout'
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-
-      - name: 'Install node.js 20.x'
-        uses: actions/setup-node@v4
-        with:
-          node-version: '20.x'
-
-      - name: 'Install dependencies (Linux amd64)'
-        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
-        run: |
-          sudo apt-fast update -qq
-          DEBIAN_FRONTEND='noninteractive' \
-            sudo apt-fast install --no-install-recommends -yq \
-              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
-              valgrind libc6-dbg libblas-dev xorg-dev
-
-      - name: 'Add build binaries to PATH'
-        shell: bash
-        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
-
-      - name: 'Build csourcesAny'
-        shell: bash
-        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
-
-      - name: 'Build koch'
-        shell: bash
-        run: nim c koch
-
-      - name: 'Build Nim'
-        shell: bash
-        run: ./koch boot -d:release -d:nimStrictMode --lib:lib
-
-      - name: 'Build Nimble'
-        shell: bash
-        run: ./koch nimble
-
-      - name: 'Action'
-        shell: bash
-        run: nim c -r -d:release ci/action.nim
-
-      - name: 'Checkout minimize'
-        uses: actions/checkout@v4
-        with:
-          repository: 'nim-lang/ci_bench'
-          path: minimize
-
-      - name: 'Run minimize benchmarks'
-        shell: bash
-        run: ./minimize/minimize ci-bench
-
-      - name: 'Restore minimize cached database'
-        uses: actions/cache/restore@v3
-        with:
-          path: minimize.csv
-          key: minimize-db-key
-
-      - name: 'Update minimize db'
-        shell: bash
-        run: ./minimize/minimize update-db
-
-      - name: 'Save minimize cached database'
-        if: |
-          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
-          matrix.target == 'linux'
-        uses: actions/cache/save@v3
-        with:
-          path: minimize.csv
-          key: minimize-db-key
-
-      - name: 'Generate minimize report'
-        shell: bash
-        run: ./minimize/minimize generate-report
-
-      - name: 'Archive minimize report'
-        uses: actions/upload-artifact@v3
-        with:
-          name: minimize-report
-          path: |
-            minimize/minimize.html
-            minimize/minimize.csv
-
-      # Requires additional permissions, see:
-      # https://github.com/nim-lang/Nim/actions/runs/4778177321/jobs/8494423792?pr=21566
-      # - name: 'Publish HTML report'
-      #   uses: rossjrw/pr-preview-action@v1
-      #   with:
-      #     source-dir: minimize
-      #     umbrella-dir: minimize
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract md summary
-        run: |
-          cat minimize/summary.md  >> $GITHUB_STEP_SUMMARY

.github/workflows/ci_docs.yml

@@ -45,7 +45,7 @@ jobs:
           - target: windows
             os: windows-2019
           - target: osx
-            os: macos-12
+            os: macos-13
 
     name: ${{ matrix.target }}
     runs-on: ${{ matrix.os }}

.github/workflows/ci_packages.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - 'devel'
+      - 'version-2-2'
       - 'version-2-0'
       - 'version-1-6'
       - 'version-1-2'
@@ -17,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-12]
+        os: [ubuntu-20.04, macos-13]
         cpu: [amd64]
         batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num`
     name: '${{ matrix.os }} (batch: ${{ matrix.batch }})'
@@ -48,6 +49,7 @@ jobs:
       - name: 'Install dependencies (macOS)'
         if: runner.os == 'macOS'
         run: brew install boehmgc make sfml gtk+3
+        # XXX can't find boehm and gtk on macos 13
       - name: 'Install dependencies (Windows)'
         if: runner.os == 'Windows'
         shell: bash

azure-pipelines.yml

@@ -20,7 +20,7 @@ jobs:
   strategy:
     matrix:
       Linux_amd64:
-        vmImage: 'ubuntu-20.04'
+        vmImage: 'ubuntu-24.04'
         CPU: amd64
       # regularly breaks, refs bug #17325
 #       Linux_i386:
@@ -29,10 +29,10 @@ jobs:
 #         vmImage: 'ubuntu-18.04'
 #         CPU: i386
       OSX_amd64:
-        vmImage: 'macOS-12'
+        vmImage: 'macOS-13'
         CPU: amd64
       OSX_amd64_cpp:
-        vmImage: 'macOS-12'
+        vmImage: 'macOS-13'
         CPU: amd64
         NIM_COMPILE_TO_CPP: true
       Windows_amd64_batch0_3:
@@ -80,10 +80,12 @@ jobs:
     - bash: |
         set -e
         . ci/funs.sh
-        echo_run sudo apt-fast update -qq
+        echo_run sudo add-apt-repository universe
+        echo_run sudo apt-get update -qq
         DEBIAN_FRONTEND='noninteractive' \
-          echo_run sudo apt-fast install --no-install-recommends -yq \
-            libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
+          echo_run sudo apt-get install --no-install-recommends -yq \
+            gcc-14 g++-14 libpcre3 liblapack-dev libpcre3 liblapack-dev libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
+        echo_run sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 60 --slave /usr/bin/g++ g++ /usr/bin/g++-14
       displayName: 'Install dependencies (amd64 Linux)'
       condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
 
@@ -100,15 +102,16 @@ jobs:
         Pin-Priority: 1001
         EOF
 
-        # echo_run sudo apt-fast update -qq
-        echo_run sudo apt-fast update -qq || echo "failed, see bug #17343"
+        # echo_run sudo apt-get update -qq
+        echo_run sudo apt-get update -qq || echo "failed, see bug #17343"
         # `:i386` (e.g. in `libffi-dev:i386`) is needed otherwise you may get:
         # `could not load: libffi.so` during dynamic loading.
         DEBIAN_FRONTEND='noninteractive' \
-          echo_run sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+          echo_run sudo apt-get install --no-install-recommends --allow-downgrades -yq \
             g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
             libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 libffi-dev:i386
 
+
         cat << EOF > bin/gcc
         #!/bin/bash
 
@@ -130,6 +133,7 @@ jobs:
     - bash: brew install boehmgc make sfml
       displayName: 'Install dependencies (OSX)'
       condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Darwin'))
+      # XXX can't find boehm on macos 13
 
     - bash: |
         set -e

build_all.bat

@@ -24,6 +24,6 @@ if not exist %nim_csources% (
   cd ..
   copy /y bin\nim.exe  %nim_csources%
 )
- bin\nim.exe c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
- koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
- koch tools --skipUserCfg --skipParentCfg --hints:off
+bin\nim.exe c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
+koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
+koch tools --skipUserCfg --skipParentCfg --hints:off

changelog.md

@@ -1,109 +1,67 @@
-# v2.2.0 - yyyy-mm-dd
+# v2.x.x - yyyy-mm-dd
 
 
 ## Changes affecting backward compatibility
 
-- `-d:nimStrictDelete` becomes the default. An index error is produced when the index passed to `system.delete` was out of bounds. Use `-d:nimAuditDelete` to mimic the old behavior for backwards compatibility.
-- The default user-agent in `std/httpclient` has been changed to `Nim-httpclient/<version>` instead of `Nim httpclient/<version>` which was incorrect according to the HTTP spec.
-- Methods now support implementations based on a VTable by using `--experimental:vtables`. Methods are then confined to be in the same module where their type has been defined.
-- With `-d:nimPreviewNonVarDestructor`, non-var destructors become the default.
-- A bug where tuple unpacking assignment with a longer tuple on the RHS than the LHS was allowed has been fixed, i.e. code like:
-  ```nim
-  var a, b: int
-  (a, b) = (1, 2, 3, 4)
-  ```
-  will no longer compile.
-- `internalNew` is removed from system, use `new` instead.
+- `-d:nimPreviewFloatRoundtrip` becomes the default. `system.addFloat` and `system.$` now can produce string representations of
+floating point numbers that are minimal in size and possess round-trip and correct
+rounding guarantees (via the
+[Dragonbox](https://raw.githubusercontent.com/jk-jeon/dragonbox/master/other_files/Dragonbox.pdf) algorithm). Use `-d:nimLegacySprintf` to emulate old behaviors.
 
-- `bindMethod` in `std/jsffi` is deprecated, don't use it with closures.
+- The `default` parameter of `tables.getOrDefault` has been renamed to `def` to
+  avoid conflicts with `system.default`, so named argument usage for this
+  parameter like `getOrDefault(..., default = ...)` will have to be changed.
 
 ## Standard library additions and changes
 
-[//]: # "Changes:"
-
-- Changed `std/osfiles.copyFile` to allow to specify `bufferSize` instead of a hardcoded one.
-- Changed `std/osfiles.copyFile` to use `POSIX_FADV_SEQUENTIAL` hints for kernel-level aggressive sequential read-aheads.
-- `std/htmlparser` has been moved to a nimble package, use `nimble` or `atlas` to install it.
-
 [//]: # "Additions:"
+- `setutils.symmetricDifference` along with its operator version
+  `` setutils.`-+-` `` and in-place version `setutils.toggle` have been added
+  to more efficiently calculate the symmetric difference of bitsets.
 
-- Added `newStringUninit` to system, which creates a new string of length `len` like `newString` but with uninitialized content.
-- Added `setLenUninit` to system, which doesn't initalize
-slots when enlarging a sequence.
-- Added `hasDefaultValue` to `std/typetraits` to check if a type has a valid default value.
-- Added Viewport API for the JavaScript targets in the `dom` module.
-- Added `toSinglyLinkedRing` and `toDoublyLinkedRing` to `std/lists` to convert from `openArray`s.
-- ORC: To be enabled via `nimOrcStats` there is a new API called `GC_orcStats` that can be used to query how many
-  objects the cyclic collector did free. If the number is zero that is a strong indicator that you can use `--mm:arc`
-  instead of `--mm:orc`.
-
-[//]: # "Deprecations:"
-
-- Deprecates `system.newSeqUninitialized`, which is replaced by `newSeqUninit`.
-
-[//]: # "Removals:"
-
+[//]: # "Changes:"
+- `std/math` The `^` symbol now supports floating-point as exponent in addition to the Natural type.
 
 ## Language changes
 
-- `noInit` can be used in types and fields to disable member initializers in the C++ backend.
-- C++ custom constructors initializers see https://nim-lang.org/docs/manual_experimental.htm#constructor-initializer
-- `member` can be used to attach a procedure to a C++ type.
-- C++ `constructor` now reuses `result` instead creating `this`.
-
-- Tuple unpacking changes:
-  - Tuple unpacking assignment now supports using underscores to discard values.
-    ```nim
-    var a, c: int
-    (a, _, c) = (1, 2, 3)
-    ```
-  - Tuple unpacking variable declarations now support type annotations, but
-    only for the entire tuple.
-    ```nim
-    let (a, b): (int, int) = (1, 2)
-    let (a, (b, c)): (byte, (float, cstring)) = (1, (2, "abc"))
-    ```
-
-- An experimental option `genericsOpenSym` has been added to allow captured
-  symbols in generic routine bodies to be replaced by symbols injected locally
-  by templates/macros at instantiation time. `bind` may be used to keep the
-  captured symbols over the injected ones regardless of enabling the option.
-
-  Since this change may affect runtime behavior, the experimental switch
-  `genericsOpenSym` needs to be enabled, and a warning is given in the case
-  where an injected symbol would replace a captured symbol not bound by `bind`
-  and the experimental switch isn't enabled.
+- An experimental option `--experimental:typeBoundOps` has been added that
+  implements the RFC https://github.com/nim-lang/RFCs/issues/380.
+  This makes the behavior of interfaces like `hash`, `$`, `==` etc. more
+  reliable for nominal types across indirect/restricted imports.
 
   ```nim
-  const value = "captured"
-  template foo(x: int, body: untyped) =
-    let value {.inject.} = "injected"
-    body
+  # objs.nim
+  import std/hashes
 
-  proc old[T](): string =
-    foo(123):
-      return value # warning: a new `value` has been injected, use `bind` or turn on `experimental:genericsOpenSym`
-  echo old[int]() # "captured"
+  type
+    Obj* = object
+      x*, y*: int
+      z*: string # to be ignored for equality
 
-  {.experimental: "genericsOpenSym".}
+  proc `==`*(a, b: Obj): bool =
+    a.x == b.x and a.y == b.y
 
-  proc bar[T](): string =
-    foo(123):
-      return value
-  assert bar[int]() == "injected" # previously it would be "captured"
-
-  proc baz[T](): string =
-    bind value
-    foo(123):
-      return value
-  assert baz[int]() == "captured"
+  proc hash*(a: Obj): Hash =
+    $!(hash(a.x) &! hash(a.y))
   ```
 
+  ```nim
+  # main.nim
+  {.experimental: "typeBoundOps".}
+  from objs import Obj # objs.hash, objs.`==` not imported
+  import std/tables
+
+  var t: Table[Obj, int]
+  t[Obj(x: 3, y: 4, z: "debug")] = 34
+  echo t[Obj(x: 3, y: 4, z: "ignored")] # 34
+  ```
+
+  See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#typeminusbound-overloads)
+  for more information.
+
 ## Compiler changes
 
-- `--nimcache` using a relative path as the argument in a config file is now relative to the config file instead of the current directory.
 
 ## Tool changes
 
-- koch now allows bootstrapping with `-d:nimHasLibFFI`, replacing the older option of building the compiler directly w/ the `libffi` nimble package in tow.
 

changelogs/changelog_2_2_0.md

@@ -1,12 +1,248 @@
-# v2.2.0 - 2023-mm-dd
+# v2.2.0 - 2024-10-02
+
 
 ## Changes affecting backward compatibility
 
+- `-d:nimStrictDelete` becomes the default. An index error is produced when the index passed to `system.delete` is out of bounds. Use `-d:nimAuditDelete` to mimic the old behavior for backward compatibility.
+
+- The default user-agent in `std/httpclient` has been changed to `Nim-httpclient/<version>` instead of `Nim httpclient/<version>` which was incorrect according to the HTTP spec.
+
+- Methods now support implementations based on a VTable by using `--experimental:vtables`. Methods are then confined to the same module where their type has been defined.
+
+- With `-d:nimPreviewNonVarDestructor`, non-var destructors become the default.
+
+- A bug where tuple unpacking assignment with a longer tuple on the RHS than the LHS was allowed has been fixed, i.e. code like:
+  ```nim
+  var a, b: int
+  (a, b) = (1, 2, 3, 4)
+  ```
+  will no longer compile.
+
+- `internalNew` is removed from the `system` module, use `new` instead.
+
+- `bindMethod` in `std/jsffi` is deprecated, don't use it with closures.
+
+- The JS backend now supports lambda lifting for closures. Use `--legacy:jsNoLambdaLifting` to emulate old behaviors.
+
+- The JS backend now supports closure iterators.
+
+- `owner` in `std/macros` is deprecated.
+
+- Ambiguous type symbols in generic procs and templates now generate symchoice nodes.
+  Previously; in templates they would error immediately at the template definition,
+  and in generic procs a type symbol would arbitrarily be captured, losing the
+  information of the other symbols. This means that generic code can now give
+  errors for ambiguous type symbols, and macros operating on generic proc AST
+  may encounter symchoice nodes instead of the arbitrarily resolved type symbol nodes.
+
+- Partial generic instantiation of routines is no longer allowed. Previously
+  it compiled in niche situations due to bugs in the compiler.
+
+  ```nim
+  proc foo[T, U](x: T, y: U) = echo (x, y)
+  proc foo[T, U](x: var T, y: U) = echo "var ", (x, y)
+
+  proc bar[T]() =
+    foo[float](1, "abc")
+
+  bar[int]() # before: (1.0, "abc"), now: type mismatch, missing generic parameter
+  ```
+
+- `const` values now open a new scope for each constant, meaning symbols
+  declared in them can no longer be used outside or in the value of
+  other constants.
+
+  ```nim
+  const foo = (var a = 1; a)
+  const bar = a # error
+  let baz = a # error
+  ```
+
+- The following POSIX wrappers have had their types changed from signed to
+  unsigned types on OSX and FreeBSD/OpenBSD to correct codegen errors:
+  - `Gid` (was `int32`, is now `uint32`)
+  - `Uid` (was `int32`, is now `uint32`)
+  - `Dev` (was `int32`, is now `uint32` on FreeBSD)
+  - `Nlink` (was `int16`, is now `uint32` on OpenBSD and `uint16` on OSX/other BSD)
+  - `sin6_flowinfo` and `sin6_scope_id` fields of `Sockaddr_in6`
+    (were `int32`, are now `uint32`)
+  - `n_net` field of `Tnetent` (was `int32`, is now `uint32`)
+
+- The `Atomic[T]` type on C++ now uses C11 primitives by default instead of
+  `std::atomic`. To use `std::atomic` instead, `-d:nimUseCppAtomics` can be defined.
+
+
+
+
+
+
 ## Standard library additions and changes
 
+[//]: # "Changes:"
+
+- Changed `std/osfiles.copyFile` to allow specifying `bufferSize` instead of a hard-coded one.
+- Changed `std/osfiles.copyFile` to use `POSIX_FADV_SEQUENTIAL` hints for kernel-level aggressive sequential read-aheads.
+- `std/htmlparser` has been moved to a nimble package, use `nimble` or `atlas` to install it.
+- Changed `std/os.copyDir` and `copyDirWithPermissions` to allow skipping special "file" objects like FIFOs, device files, etc on Unix by specifying a `skipSpecial` parameter.
+
+[//]: # "Additions:"
+
+- Added `newStringUninit` to the `system` module, which creates a new string of length `len` like `newString` but with uninitialized content.
+- Added `setLenUninit` to the `system` module, which doesn't initialize
+slots when enlarging a sequence.
+- Added `hasDefaultValue` to `std/typetraits` to check if a type has a valid default value.
+- Added `rangeBase` to `std/typetraits` to obtain the base type of a range type or
+  convert a value with a range type to its base type.
+- Added Viewport API for the JavaScript targets in the `dom` module.
+- Added `toSinglyLinkedRing` and `toDoublyLinkedRing` to `std/lists` to convert from `openArray`s.
+- ORC: To be enabled via `nimOrcStats` there is a new API called `GC_orcStats` that can be used to query how many
+  objects the cyclic collector did free. If the number is zero that is a strong indicator that you can use `--mm:arc`
+  instead of `--mm:orc`.
+- A `$` template is provided for `Path` in `std/paths`.
+- `std/hashes.hash(x:string)` changed to produce a 64-bit string `Hash` (based
+on Google's Farm Hash) which is also often faster than the present one.  Define
+`nimStringHash2` to get the old values back.  `--jsbigint=off` mode always only
+produces the old values.  This may impact your automated tests if they depend
+on hash order in some obvious or indirect way.  Using `sorted` or `OrderedTable`
+is often an easy workaround.
+
+[//]: # "Deprecations:"
+
+- Deprecates `system.newSeqUninitialized`, which is replaced by `newSeqUninit`.
+
+[//]: # "Removals:"
+
+
+
+
+
+
 ## Language changes
 
+- `noInit` can be used in types and fields to disable member initializers in the C++ backend.
+
+- C++ custom constructors initializers see https://nim-lang.org/docs/manual_experimental.html#constructor-initializer
+
+- `member` can be used to attach a procedure to a C++ type.
+
+- Inside a C++ constructor, `result` can be used to access the created object rather than `this`.
+
+- Tuple unpacking changes:
+  - Tuple unpacking assignment now supports using underscores to discard values.
+    ```nim
+    var a, c: int
+    (a, _, c) = (1, 2, 3)
+    ```
+  - Tuple unpacking variable declarations now support type annotations, but
+    only for the entire tuple.
+    ```nim
+    let (a, b): (int, int) = (1, 2)
+    let (a, (b, c)): (byte, (float, cstring)) = (1, (2, "abc"))
+    ```
+
+- The experimental option `--experimental:openSym` has been added to allow
+  captured symbols in generic routine and template bodies respectively to be
+  replaced by symbols injected locally by templates/macros at instantiation
+  time. `bind` may be used to keep the captured symbols over the injected ones
+  regardless of enabling the option, but other methods like renaming the
+  captured symbols should be used instead so that the code is not affected by
+  context changes.
+
+  Since this change may affect runtime behavior, the experimental switch
+  `openSym` needs to be enabled; and a warning is given in the case where an
+  injected symbol would replace a captured symbol not bound by `bind` and
+  the experimental switch isn't enabled.
+
+  ```nim
+  const value = "captured"
+  template foo(x: int, body: untyped): untyped =
+    let value {.inject.} = "injected"
+    body
+
+  proc old[T](): string =
+    foo(123):
+      return value # warning: a new `value` has been injected, use `bind` or turn on `experimental:openSym`
+  echo old[int]() # "captured"
+
+  template oldTempl(): string =
+    block:
+      foo(123):
+        value # warning: a new `value` has been injected, use `bind` or turn on `experimental:openSym`
+  echo oldTempl() # "captured"
+
+  {.experimental: "openSym".}
+
+  proc bar[T](): string =
+    foo(123):
+      return value
+  assert bar[int]() == "injected" # previously it would be "captured"
+
+  proc baz[T](): string =
+    bind value
+    foo(123):
+      return value
+  assert baz[int]() == "captured"
+
+  template barTempl(): string =
+    block:
+      foo(123):
+        value
+  assert barTempl() == "injected" # previously it would be "captured"
+
+  template bazTempl(): string =
+    bind value
+    block:
+      foo(123):
+        value
+  assert bazTempl() == "captured"
+  ```
+
+  This option also generates a new node kind `nnkOpenSym` which contains
+  exactly 1 `nnkSym` node. In the future this might be merged with a slightly
+  modified `nnkOpenSymChoice` node but macros that want to support the
+  experimental feature should still handle `nnkOpenSym`, as the node kind would
+  simply not be generated as opposed to being removed.
+
+  Another experimental switch `genericsOpenSym` exists that enables this behavior
+  at instantiation time, meaning templates etc can enable it specifically when
+  they are being called. However this does not generate `nnkOpenSym` nodes
+  (unless the other switch is enabled) and so doesn't reflect the regular
+  behavior of the switch.
+
+  ```nim
+  const value = "captured"
+  template foo(x: int, body: untyped): untyped =
+    let value {.inject.} = "injected"
+    {.push experimental: "genericsOpenSym".}
+    body
+    {.pop.}
+
+  proc bar[T](): string =
+    foo(123):
+      return value
+  echo bar[int]() # "injected"
+
+  template barTempl(): string =
+    block:
+      var res: string
+      foo(123):
+        res = value
+      res
+  assert barTempl() == "injected"
+  ```
+
+
+
+
+
 ## Compiler changes
 
+- `--nimcache` using a relative path as the argument in a config file is now relative to the config file instead of the current directory.
+
+
+
+
+
 ## Tool changes
 
+- koch now allows bootstrapping with `-d:nimHasLibFFI`, replacing the older option of building the compiler directly w/ the `libffi` nimble package.

changelogs/changelog_X_XX_X.md

@@ -1,4 +1,4 @@
-# v1.xx.x - yyyy-mm-dd
+# v2.xx.x - yyyy-mm-dd
 
 This is an example file.
 The changes should go to changelog.md!

compiler/ast.nim

@@ -41,7 +41,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # 51 flags!
+  TSymFlag* = enum    # 63 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -93,7 +93,7 @@ type
     sfNamedParamCall, # symbol needs named parameter call syntax in target
                       # language; for interfacing with Objective C
     sfDiscardable,    # returned value may be discarded implicitly
-    sfOverridden,      # proc is overridden
+    sfOverridden,     # proc is overridden
     sfCallsite        # A flag for template symbols to tell the
                       # compiler it should use line information from
                       # the calling side of the macro, not from the
@@ -126,24 +126,25 @@ type
     sfByCopy          # param is marked as pass bycopy
     sfMember          # proc is a C++ member of a type
     sfCodegenDecl     # type, proc, global or proc param is marked as codegenDecl
+    sfWasGenSym       # symbol was 'gensym'ed
+    sfForceLift       # variable has to be lifted into closure environment
+
+    sfDirty           # template is not hygienic (old styled template) module,
+                      # compiled from a dirty-buffer
+    sfCustomPragma    # symbol is custom pragma template
+    sfBase,           # a base method
+    sfGoto            # var is used for 'goto' code generation
+    sfAnon,           # symbol name that was generated by the compiler
+                      # the compiler will avoid printing such names
+                      # in user messages.
+    sfAllUntyped      # macro or template is immediately expanded in a generic context
+    sfTemplateRedefinition # symbol is a redefinition of an earlier template
 
   TSymFlags* = set[TSymFlag]
 
 const
   sfNoInit* = sfMainModule       # don't generate code to init the variable
 
-  sfAllUntyped* = sfVolatile # macro or template is immediately expanded \
-    # in a generic context
-
-  sfDirty* = sfPure
-    # template is not hygienic (old styled template)
-    # module, compiled from a dirty-buffer
-
-  sfAnon* = sfDiscardable
-    # symbol name that was generated by the compiler
-    # the compiler will avoid printing such names
-    # in user messages.
-
   sfNoForward* = sfRegister
     # forward declarations are not required (per module)
   sfReorder* = sfForward
@@ -152,12 +153,8 @@ const
   sfCompileToCpp* = sfInfixCall       # compile the module as C++ code
   sfCompileToObjc* = sfNamedParamCall # compile the module as Objective-C code
   sfExperimental* = sfOverridden       # module uses the .experimental switch
-  sfGoto* = sfOverridden               # var is used for 'goto' code generation
   sfWrittenTo* = sfBorrow             # param is assigned to
                                       # currently unimplemented
-  sfBase* = sfDiscriminant
-  sfCustomPragma* = sfRegister        # symbol is custom pragma template
-  sfTemplateRedefinition* = sfExportc # symbol is a redefinition of an earlier template
   sfCppMember* = { sfVirtual, sfMember, sfConstructor } # proc is a C++ member, meaning it will be attached to the type definition
 
 const
@@ -217,7 +214,8 @@ type
     tyUncheckedArray
       # An array with boundaries [0,+∞]
 
-    tyProxy # used as errornous type (for idetools)
+    tyError # used as erroneous type (for idetools)
+      # as an erroneous node should match everything
 
     tyBuiltInTypeClass
       # Type such as the catch-all object, tuple, seq, etc
@@ -279,10 +277,6 @@ static:
 const
   tyPureObject* = tyTuple
   GcTypeKinds* = {tyRef, tySequence, tyString}
-  tyError* = tyProxy # as an errornous node should match everything
-  tyUnknown* = tyFromExpr
-
-  tyUnknownTypes* = {tyError, tyFromExpr}
 
   tyTypeClasses* = {tyBuiltInTypeClass, tyCompositeTypeClass,
                     tyUserTypeClass, tyUserTypeClassInst,
@@ -328,7 +322,9 @@ type
     nfFirstWrite # this node is a first write
     nfHasComment # node has a comment
     nfSkipFieldChecking # node skips field visable checking
-    nfOpenSym # node is a captured sym but can be overriden by local symbols
+    nfDisabledOpenSym # temporary: node should be nkOpenSym but cannot
+                      # because openSym experimental switch is disabled
+                      # gives warning instead
 
   TNodeFlags* = set[TNodeFlag]
   TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 47)
@@ -449,6 +445,8 @@ const
   tfGcSafe* = tfThread
   tfObjHasKids* = tfEnumHasHoles
   tfReturnsNew* = tfInheritable
+  tfNonConstExpr* = tfExplicitCallConv
+    ## tyFromExpr where the expression shouldn't be evaluated as a static value
   skError* = skUnknown
 
 var
@@ -493,7 +491,7 @@ type
     mAnd, mOr,
     mImplies, mIff, mExists, mForall, mOld,
     mEqStr, mLeStr, mLtStr,
-    mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet,
+    mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mXorSet,
     mConStrStr, mSlice,
     mDotDot, # this one is only necessary to give nice compile time warnings
     mFields, mFieldPairs, mOmpParFor,
@@ -504,7 +502,7 @@ type
     mSwap, mIsNil, mArrToSeq, mOpenArrayToSeq,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
     mMove, mEnsureMove, mWasMoved, mDup, mDestroy, mTrace,
-    mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mAccessTypeField, mReset,
+    mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mAccessTypeField,
     mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
     mOrdinal, mIterableType,
@@ -561,7 +559,7 @@ const
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mEqStr, mLeStr, mLtStr,
-    mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet,
+    mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mXorSet,
     mConStrStr, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
     mInSet, mRepr, mOpenArrayToSeq}
 
@@ -593,7 +591,7 @@ type
   TNode*{.final, acyclic.} = object # on a 32bit machine, this takes 32 bytes
     when defined(useNodeIds):
       id*: int
-    typ*: PType
+    typField: PType
     info*: TLineInfo
     flags*: TNodeFlags
     case kind*: TNodeKind
@@ -654,7 +652,7 @@ type
     storage*: TStorageLoc
     flags*: TLocFlags         # location's flags
     lode*: PNode              # Node where the location came from; can be faked
-    r*: Rope                  # rope value of location (code generators)
+    snippet*: Rope            # C code snippet of location (code generators)
 
   # ---------------- end of backend information ------------------------------
 
@@ -708,7 +706,7 @@ type
     when defined(nimsuggest):
       endInfo*: TLineInfo
       hasUserSpecifiedType*: bool  # used for determining whether to display inlay type hints
-    owner*: PSym
+    ownerField: PSym
     flags*: TSymFlags
     ast*: PNode               # syntax tree of proc, iterator, etc.:
                               # the whole proc including header; this is used
@@ -777,7 +775,7 @@ type
                               # formal param list
                               # for concepts, the concept body
                               # else: unused
-    owner*: PSym              # the 'owner' of the type
+    ownerField: PSym          # the 'owner' of the type
     sym*: PSym                # types have the sym associated with them
                               # it is used for converting types to strings
     size*: BiggestInt         # the size of the type in bytes
@@ -816,6 +814,15 @@ type
 
 template nodeId(n: PNode): int = cast[int](n)
 
+template typ*(n: PNode): PType =
+  n.typField
+
+proc owner*(s: PSym|PType): PSym {.inline.} =
+  result = s.ownerField
+
+proc setOwner*(s: PSym|PType, owner: PSym) {.inline.} =
+  s.ownerField = owner
+
 type Gconfig = object
   # we put comments in a side channel to avoid increasing `sizeof(TNode)`, which
   # reduces memory usage given that `PNode` is the most allocated type by far.
@@ -883,7 +890,7 @@ const
                                       nfFromTemplate, nfDefaultRefsParam,
                                       nfExecuteOnReload, nfLastRead,
                                       nfFirstWrite, nfSkipFieldChecking,
-                                      nfOpenSym}
+                                      nfDisabledOpenSym}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -897,7 +904,7 @@ const
   nfAllFieldsSet* = nfBase2
 
   nkIdentKinds* = {nkIdent, nkSym, nkAccQuoted, nkOpenSymChoice,
-                   nkClosedSymChoice}
+                   nkClosedSymChoice, nkOpenSym}
 
   nkPragmaCallKinds* = {nkExprColonExpr, nkCall, nkCallStrLit}
   nkLiterals* = {nkCharLit..nkTripleStrLit}
@@ -925,6 +932,7 @@ proc getPIdent*(a: PNode): PIdent {.inline.} =
   of nkSym: a.sym.name
   of nkIdent: a.ident
   of nkOpenSymChoice, nkClosedSymChoice: a.sons[0].sym.name
+  of nkOpenSym: getPIdent(a.sons[0])
   else: nil
 
 const
@@ -1058,8 +1066,11 @@ proc getDeclPragma*(n: PNode): PNode =
 
 proc extractPragma*(s: PSym): PNode =
   ## gets the pragma node of routine/type/var/let/const symbol `s`
-  if s.kind in routineKinds:
-    result = s.ast[pragmasPos]
+  if s.kind in routineKinds: # bug #24167
+    if s.ast[pragmasPos] != nil and s.ast[pragmasPos].kind != nkEmpty:
+      result = s.ast[pragmasPos]
+    else:
+      result = nil
   elif s.kind in {skType, skVar, skLet, skConst}:
     if s.ast != nil and s.ast.len > 0:
       if s.ast[0].kind == nkPragmaExpr and s.ast[0].len > 1:
@@ -1130,7 +1141,7 @@ proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
   ## new node with line info, type, and no children
   result = newNode(kind)
   result.info = info
-  result.typ = typ
+  result.typ() = typ
 
 proc newNode*(kind: TNodeKind, info: TLineInfo): PNode =
   ## new node with line info, no type, and no children
@@ -1195,7 +1206,7 @@ proc newSym*(symKind: TSymKind, name: PIdent, idgen: IdGenerator; owner: PSym,
   assert not name.isNil
   let id = nextSymId idgen
   result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id,
-                options: options, owner: owner, offset: defaultOffset,
+                options: options, ownerField: owner, offset: defaultOffset,
                 disamb: getOrDefault(idgen.disambTable, name).int32)
   idgen.disambTable.inc name
   when false:
@@ -1276,15 +1287,18 @@ proc newIdentNode*(ident: PIdent, info: TLineInfo): PNode =
 proc newSymNode*(sym: PSym): PNode =
   result = newNode(nkSym)
   result.sym = sym
-  result.typ = sym.typ
+  result.typ() = sym.typ
   result.info = sym.info
 
 proc newSymNode*(sym: PSym, info: TLineInfo): PNode =
   result = newNode(nkSym)
   result.sym = sym
-  result.typ = sym.typ
+  result.typ() = sym.typ
   result.info = info
 
+proc newOpenSym*(n: PNode): PNode {.inline.} =
+  result = newTreeI(nkOpenSym, n.info, n)
+
 proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode =
   result = newNode(kind)
   result.intVal = intVal
@@ -1359,7 +1373,7 @@ proc newIntTypeNode*(intVal: BiggestInt, typ: PType): PNode =
     result = newNode(nkIntLit)
   else: raiseAssert $kind
   result.intVal = intVal
-  result.typ = typ
+  result.typ() = typ
 
 proc newIntTypeNode*(intVal: Int128, typ: PType): PNode =
   # XXX: introduce range check
@@ -1498,7 +1512,7 @@ iterator signature*(t: PType): PType =
 
 proc newType*(kind: TTypeKind; idgen: IdGenerator; owner: PSym; son: sink PType = nil): PType =
   let id = nextTypeId idgen
-  result = PType(kind: kind, owner: owner, size: defaultSize,
+  result = PType(kind: kind, ownerField: owner, size: defaultSize,
                  align: defaultAlignment, itemId: id,
                  uniqueId: id, sons: @[])
   if son != nil: result.sons.add son
@@ -1509,13 +1523,14 @@ proc newType*(kind: TTypeKind; idgen: IdGenerator; owner: PSym; son: sink PType
 
 proc setSons*(dest: PType; sons: sink seq[PType]) {.inline.} = dest.sons = sons
 proc setSon*(dest: PType; son: sink PType) {.inline.} = dest.sons = @[son]
+proc setSonsLen*(dest: PType; len: int) {.inline.} = setLen(dest.sons, len)
 
 proc mergeLoc(a: var TLoc, b: TLoc) =
   if a.k == low(typeof(a.k)): a.k = b.k
   if a.storage == low(typeof(a.storage)): a.storage = b.storage
   a.flags.incl b.flags
   if a.lode == nil: a.lode = b.lode
-  if a.r == "": a.r = b.r
+  if a.snippet == "": a.snippet = b.snippet
 
 proc newSons*(father: PNode, length: int) =
   setLen(father.sons, length)
@@ -1552,7 +1567,7 @@ proc copyType*(t: PType, idgen: IdGenerator, owner: PSym): PType =
   result.sym = t.sym          # backend-info should not be copied
 
 proc exactReplica*(t: PType): PType =
-  result = PType(kind: t.kind, owner: t.owner, size: defaultSize,
+  result = PType(kind: t.kind, ownerField: t.owner, size: defaultSize,
                  align: defaultAlignment, itemId: t.itemId,
                  uniqueId: t.uniqueId)
   assignType(result, t)
@@ -1630,7 +1645,7 @@ proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) =
   if mask != {} and propagateHasAsgn:
     let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink})
     if o2.kind in {tyTuple, tyObject, tyArray,
-                   tySequence, tySet, tyDistinct}:
+                   tySequence, tyString, tySet, tyDistinct}:
       o2.flags.incl mask
       owner.flags.incl mask
 
@@ -1660,7 +1675,7 @@ proc copyNode*(src: PNode): PNode =
     return nil
   result = newNode(src.kind)
   result.info = src.info
-  result.typ = src.typ
+  result.typ() = src.typ
   result.flags = src.flags * PersistentNodeFlags
   result.comment = src.comment
   when defined(useNodeIds):
@@ -1678,7 +1693,7 @@ proc copyNode*(src: PNode): PNode =
 
 template transitionNodeKindCommon(k: TNodeKind) =
   let obj {.inject.} = n[]
-  n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags)
+  n[] = TNode(kind: k, typField: n.typ, info: obj.info, flags: obj.flags)
   # n.comment = obj.comment # shouldn't be needed, the address doesnt' change
   when defined(useNodeIds):
     n.id = obj.id
@@ -1701,7 +1716,7 @@ proc transitionNoneToSym*(n: PNode) =
 template transitionSymKindCommon*(k: TSymKind) =
   let obj {.inject.} = s[]
   s[] = TSym(kind: k, itemId: obj.itemId, magic: obj.magic, typ: obj.typ, name: obj.name,
-             info: obj.info, owner: obj.owner, flags: obj.flags, ast: obj.ast,
+             info: obj.info, ownerField: obj.ownerField, flags: obj.flags, ast: obj.ast,
              options: obj.options, position: obj.position, offset: obj.offset,
              loc: obj.loc, annex: obj.annex, constraint: obj.constraint)
   when hasFFI:
@@ -1729,7 +1744,7 @@ template copyNodeImpl(dst, src, processSonsStmt) =
   dst.info = src.info
   when defined(nimsuggest):
     result.endInfo = src.endInfo
-  dst.typ = src.typ
+  dst.typ() = src.typ
   dst.flags = src.flags * PersistentNodeFlags
   dst.comment = src.comment
   when defined(useNodeIds):
@@ -1887,7 +1902,7 @@ proc skipGenericOwner*(s: PSym): PSym =
 
 proc originatingModule*(s: PSym): PSym =
   result = s
-  while result.kind != skModule: result = result.owner
+  while result != nil and result.kind != skModule: result = result.owner
 
 proc isRoutine*(s: PSym): bool {.inline.} =
   result = s.kind in skProcKinds

compiler/astyaml.nim

@@ -75,7 +75,7 @@ proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet;
     res.addf("\n$1storage: $2", [istr, makeYamlString($n.loc.storage)])
     if card(n.loc.flags) > 0:
       res.addf("\n$1flags: $2", [istr, makeYamlString($n.loc.flags)])
-    res.addf("\n$1r: $2", [istr, n.loc.r])
+    res.addf("\n$1snippet: $2", [istr, n.loc.snippet])
     res.addf("\n$1lode: $2", [istr])
     res.treeToYamlAux(conf, n.loc.lode, marker, indent + 1, maxRecDepth - 1)
 

compiler/backendpragmas.nim

@@ -1,25 +0,0 @@
-import pragmas, options, ast, trees
-
-proc pushBackendOption(optionsStack: var seq[TOptions], options: var TOptions) =
-  optionsStack.add options
-
-proc popBackendOption(optionsStack: var seq[TOptions], options: var TOptions) =
-  options = optionsStack[^1]
-  optionsStack.setLen(optionsStack.len-1)
-
-proc processPushBackendOption*(optionsStack: var seq[TOptions], options: var TOptions,
-                           n: PNode, start: int) =
-  pushBackendOption(optionsStack, options)
-  for i in start..<n.len:
-    let it = n[i]
-    if it.kind in nkPragmaCallKinds and it.len == 2 and it[1].kind == nkIntLit:
-      let sw = whichPragma(it[0])
-      let opts = pragmaToOptions(sw)
-      if opts != {}:
-        if it[1].intVal != 0:
-          options.incl opts
-        else:
-          options.excl opts
-
-template processPopBackendOption*(optionsStack: var seq[TOptions], options: var TOptions) =
-  popBackendOption(optionsStack, options)

compiler/cbuilderbase.nim

@@ -0,0 +1,88 @@
+import ropes, int128
+
+type
+  Snippet* = string
+  Builder* = object
+    buf*: string
+
+template newBuilder*(s: string): Builder =
+  Builder(buf: s)
+
+proc extract*(builder: Builder): Snippet =
+  builder.buf
+
+proc add*(builder: var Builder, s: string) =
+  builder.buf.add(s)
+
+proc add*(builder: var Builder, s: char) =
+  builder.buf.add(s)
+
+proc addIntValue*(builder: var Builder, val: int) =
+  builder.buf.addInt(val)
+
+proc addIntValue*(builder: var Builder, val: int64) =
+  builder.buf.addInt(val)
+
+proc addIntValue*(builder: var Builder, val: uint64) =
+  builder.buf.addInt(val)
+
+proc addIntValue*(builder: var Builder, val: Int128) =
+  builder.buf.addInt128(val)
+
+template cIntValue*(val: int): Snippet = $val
+template cIntValue*(val: int64): Snippet = $val
+template cIntValue*(val: uint64): Snippet = $val
+template cIntValue*(val: Int128): Snippet = $val
+
+import std/formatfloat
+
+proc addFloatValue*(builder: var Builder, val: float) =
+  builder.buf.addFloat(val)
+
+template cFloatValue*(val: float): Snippet = $val
+
+proc addInt64Literal*(result: var Builder; i: BiggestInt) =
+  if i > low(int64):
+    result.add "IL64($1)" % [rope(i)]
+  else:
+    result.add "(IL64(-9223372036854775807) - IL64(1))"
+
+proc addUint64Literal*(result: var Builder; i: uint64) =
+  result.add rope($i & "ULL")
+
+proc addIntLiteral*(result: var Builder; i: BiggestInt) =
+  if i > low(int32) and i <= high(int32):
+    result.addIntValue(i)
+  elif i == low(int32):
+    # Nim has the same bug for the same reasons :-)
+    result.add "(-2147483647 -1)"
+  elif i > low(int64):
+    result.add "IL64($1)" % [rope(i)]
+  else:
+    result.add "(IL64(-9223372036854775807) - IL64(1))"
+
+proc addIntLiteral*(result: var Builder; i: Int128) =
+  addIntLiteral(result, toInt64(i))
+
+proc cInt64Literal*(i: BiggestInt): Snippet =
+  if i > low(int64):
+    result = "IL64($1)" % [rope(i)]
+  else:
+    result = "(IL64(-9223372036854775807) - IL64(1))"
+
+proc cUint64Literal*(i: uint64): Snippet =
+  result = $i & "ULL"
+
+proc cIntLiteral*(i: BiggestInt): Snippet =
+  if i > low(int32) and i <= high(int32):
+    result = rope(i)
+  elif i == low(int32):
+    # Nim has the same bug for the same reasons :-)
+    result = "(-2147483647 -1)"
+  elif i > low(int64):
+    result = "IL64($1)" % [rope(i)]
+  else:
+    result = "(IL64(-9223372036854775807) - IL64(1))"
+
+proc cIntLiteral*(i: Int128): Snippet =
+  result = cIntLiteral(toInt64(i))

compiler/cbuilderdecls.nim

@@ -0,0 +1,527 @@
+type VarKind = enum
+  Local
+  Global
+  Threadvar
+  Const
+  AlwaysConst ## const even on C++
+
+proc addVarHeader(builder: var Builder, kind: VarKind) =
+  ## adds modifiers for given var kind:
+  ## Local has no modifier
+  ## Global has `static` modifier
+  ## Const has `static NIM_CONST` modifier
+  ## AlwaysConst has `static const` modifier (NIM_CONST is no-op on C++)
+  ## Threadvar is unimplemented
+  case kind
+  of Local: discard
+  of Global:
+    builder.add("static ")
+  of Const:
+    builder.add("static NIM_CONST ")
+  of AlwaysConst:
+    builder.add("static const ")
+  of Threadvar:
+    doAssert false, "unimplemented"
+
+proc addVar(builder: var Builder, kind: VarKind = Local, name: string, typ: Snippet, initializer: Snippet = "") =
+  ## adds a variable declaration to the builder
+  builder.addVarHeader(kind)
+  builder.add(typ)
+  builder.add(" ")
+  builder.add(name)
+  if initializer.len != 0:
+    builder.add(" = ")
+    builder.add(initializer)
+  builder.add(";\n")
+
+template addVarWithType(builder: var Builder, kind: VarKind = Local, name: string, body: typed) =
+  ## adds a variable declaration to the builder, with the `body` building the type
+  builder.addVarHeader(kind)
+  body
+  builder.add(" ")
+  builder.add(name)
+  builder.add(";\n")
+
+template addVarWithInitializer(builder: var Builder, kind: VarKind = Local, name: string,
+                               typ: Snippet, initializerBody: typed) =
+  ## adds a variable declaration to the builder, with
+  ## `initializerBody` building the initializer. initializer must be provided
+  builder.addVarHeader(kind)
+  builder.add(typ)
+  builder.add(" ")
+  builder.add(name)
+  builder.add(" = ")
+  initializerBody
+  builder.add(";\n")
+
+template addVarWithTypeAndInitializer(builder: var Builder, kind: VarKind = Local, name: string,
+                                      typeBody, initializerBody: typed) =
+  ## adds a variable declaration to the builder, with `typeBody` building the type, and
+  ## `initializerBody` building the initializer. initializer must be provided
+  builder.addVarHeader(kind)
+  typeBody
+  builder.add(" ")
+  builder.add(name)
+  builder.add(" = ")
+  initializerBody
+  builder.add(";\n")
+
+proc addArrayVar(builder: var Builder, kind: VarKind = Local, name: string, elementType: Snippet, len: int, initializer: Snippet = "") =
+  ## adds an array variable declaration to the builder
+  builder.addVarHeader(kind)
+  builder.add(elementType)
+  builder.add(" ")
+  builder.add(name)
+  builder.add("[")
+  builder.addIntValue(len)
+  builder.add("]")
+  if initializer.len != 0:
+    builder.add(" = ")
+    builder.add(initializer)
+  builder.add(";\n")
+
+template addArrayVarWithInitializer(builder: var Builder, kind: VarKind = Local, name: string, elementType: Snippet, len: int, body: typed) =
+  ## adds an array variable declaration to the builder with the initializer built according to `body`
+  builder.addVarHeader(kind)
+  builder.add(elementType)
+  builder.add(" ")
+  builder.add(name)
+  builder.add("[")
+  builder.addIntValue(len)
+  builder.add("] = ")
+  body
+  builder.add(";\n")
+
+template addTypedef(builder: var Builder, name: string, typeBody: typed) =
+  ## adds a typedef declaration to the builder with name `name` and type as
+  ## built in `typeBody`
+  builder.add("typedef ")
+  typeBody
+  builder.add(" ")
+  builder.add(name)
+  builder.add(";\n")
+
+proc addProcTypedef(builder: var Builder, callConv: TCallingConvention, name: string, rettype, params: Snippet) =
+  builder.add("typedef ")
+  builder.add(CallingConvToStr[callConv])
+  builder.add("_PTR(")
+  builder.add(rettype)
+  builder.add(", ")
+  builder.add(name)
+  builder.add(")")
+  builder.add(params)
+  builder.add(";\n")
+
+template addArrayTypedef(builder: var Builder, name: string, len: BiggestInt, typeBody: typed) =
+  ## adds an array typedef declaration to the builder with name `name`,
+  ## length `len`, and element type as built in `typeBody`
+  builder.add("typedef ")
+  typeBody
+  builder.add(" ")
+  builder.add(name)
+  builder.add("[")
+  builder.addIntValue(len)
+  builder.add("];\n")
+
+type
+  StructInitializerKind = enum
+    siOrderedStruct ## struct constructor, but without named fields on C
+    siNamedStruct ## struct constructor, with named fields i.e. C99 designated initializer
+    siArray ## array constructor
+    siWrapper ## wrapper for a single field, generates it verbatim
+
+  StructInitializer = object
+    ## context for building struct initializers, i.e. `{ field1, field2 }`
+    kind: StructInitializerKind
+      ## if true, fields will not be named, instead values are placed in order
+    needsComma: bool
+
+proc initStructInitializer(builder: var Builder, kind: StructInitializerKind): StructInitializer =
+  ## starts building a struct initializer, i.e. braced initializer list
+  result = StructInitializer(kind: kind, needsComma: false)
+  if kind != siWrapper:
+    builder.add("{")
+
+template addField(builder: var Builder, constr: var StructInitializer, name: string, valueBody: typed) =
+  ## adds a field to a struct initializer, with the value built in `valueBody`
+  if constr.needsComma:
+    assert constr.kind != siWrapper, "wrapper constructor cannot have multiple fields"
+    builder.add(", ")
+  else:
+    constr.needsComma = true
+  case constr.kind
+  of siArray, siWrapper:
+    # no name, can just add value
+    valueBody
+  of siOrderedStruct:
+    # no name, can just add value on C
+    assert name.len != 0, "name has to be given for struct initializer field"
+    valueBody
+  of siNamedStruct:
+    assert name.len != 0, "name has to be given for struct initializer field"
+    builder.add(".")
+    builder.add(name)
+    builder.add(" = ")
+    valueBody
+
+proc finishStructInitializer(builder: var Builder, constr: StructInitializer) =
+  ## finishes building a struct initializer
+  if constr.kind != siWrapper:
+    builder.add("}")
+
+template addStructInitializer(builder: var Builder, constr: out StructInitializer, kind: StructInitializerKind, body: typed) =
+  ## builds a struct initializer, i.e. `{ field1, field2 }`
+  ## a `var StructInitializer` must be declared and passed as a parameter so
+  ## that it can be used with `addField`
+  constr = builder.initStructInitializer(kind)
+  body
+  builder.finishStructInitializer(constr)
+
+proc addField(obj: var Builder; name, typ: Snippet; isFlexArray: bool = false; initializer: Snippet = "") =
+  ## adds a field inside a struct/union type
+  obj.add('\t')
+  obj.add(typ)
+  obj.add(" ")
+  obj.add(name)
+  if isFlexArray:
+    obj.add("[SEQ_DECL_SIZE]")
+  if initializer.len != 0:
+    obj.add(initializer)
+  obj.add(";\n")
+
+proc addArrayField(obj: var Builder; name, elementType: Snippet; len: int; initializer: Snippet = "") =
+  ## adds an array field inside a struct/union type
+  obj.add('\t')
+  obj.add(elementType)
+  obj.add(" ")
+  obj.add(name)
+  obj.add("[")
+  obj.addIntValue(len)
+  obj.add("]")
+  if initializer.len != 0:
+    obj.add(initializer)
+  obj.add(";\n")
+
+proc addField(obj: var Builder; field: PSym; name, typ: Snippet; isFlexArray: bool = false; initializer: Snippet = "") =
+  ## adds an field inside a struct/union type, based on an `skField` symbol
+  obj.add('\t')
+  if field.alignment > 0:
+    obj.add("NIM_ALIGN(")
+    obj.addIntValue(field.alignment)
+    obj.add(") ")
+  obj.add(typ)
+  if sfNoalias in field.flags:
+    obj.add(" NIM_NOALIAS")
+  obj.add(" ")
+  obj.add(name)
+  if isFlexArray:
+    obj.add("[SEQ_DECL_SIZE]")
+  if field.bitsize != 0:
+    obj.add(":")
+    obj.addIntValue(field.bitsize)
+  if initializer.len != 0:
+    obj.add(initializer)
+  obj.add(";\n")
+
+proc addProcField(obj: var Builder, callConv: TCallingConvention, name: string, rettype, params: Snippet) =
+  obj.add(CallingConvToStr[callConv])
+  obj.add("_PTR(")
+  obj.add(rettype)
+  obj.add(", ")
+  obj.add(name)
+  obj.add(")")
+  obj.add(params)
+  obj.add(";\n")
+
+type
+  BaseClassKind = enum
+    ## denotes how and whether or not the base class/RTTI should be stored
+    bcNone, bcCppInherit, bcSupField, bcNoneRtti, bcNoneTinyRtti
+  StructBuilderInfo = object
+    ## context for building `struct` types
+    baseKind: BaseClassKind
+    named: bool
+    preFieldsLen: int
+
+proc structOrUnion(t: PType): Snippet =
+  let t = t.skipTypes({tyAlias, tySink})
+  if tfUnion in t.flags: "union"
+  else: "struct"
+
+proc startSimpleStruct(obj: var Builder; m: BModule; name: string; baseType: Snippet): StructBuilderInfo =
+  result = StructBuilderInfo(baseKind: bcNone, named: name.len != 0)
+  obj.add("struct")
+  if result.named:
+    obj.add(" ")
+    obj.add(name)
+  if baseType.len != 0:
+    if m.compileToCpp:
+      result.baseKind = bcCppInherit
+    else:
+      result.baseKind = bcSupField
+  if result.baseKind == bcCppInherit:
+    obj.add(" : public ")
+    obj.add(baseType)
+  obj.add(" ")
+  obj.add("{\n")
+  result.preFieldsLen = obj.buf.len
+  if result.baseKind == bcSupField:
+    obj.addField(name = "Sup", typ = baseType)
+
+proc finishSimpleStruct(obj: var Builder; m: BModule; info: StructBuilderInfo) =
+  if info.baseKind == bcNone and info.preFieldsLen == obj.buf.len:
+    # no fields were added, add dummy field
+    obj.addField(name = "dummy", typ = "char")
+  if info.named:
+    obj.add("};\n")
+  else:
+    obj.add("}")
+
+template addSimpleStruct(obj: var Builder; m: BModule; name: string; baseType: Snippet; body: typed) =
+  ## builds a struct type not based on a Nim type with fields according to `body`,
+  ## `name` can be empty to build as a type expression and not a statement
+  let info = startSimpleStruct(obj, m, name, baseType)
+  body
+  finishSimpleStruct(obj, m, info)
+
+proc startStruct(obj: var Builder; m: BModule; t: PType; name: string; baseType: Snippet): StructBuilderInfo =
+  result = StructBuilderInfo(baseKind: bcNone, named: name.len != 0)
+  if tfPacked in t.flags:
+    if hasAttribute in CC[m.config.cCompiler].props:
+      obj.add(structOrUnion(t))
+      obj.add(" __attribute__((__packed__))")
+    else:
+      obj.add("#pragma pack(push, 1)\n")
+      obj.add(structOrUnion(t))
+  else:
+    obj.add(structOrUnion(t))
+  if result.named:
+    obj.add(" ")
+    obj.add(name)
+  if t.kind == tyObject:
+    if t.baseClass == nil:
+      if lacksMTypeField(t):
+        result.baseKind = bcNone
+      elif optTinyRtti in m.config.globalOptions:
+        result.baseKind = bcNoneTinyRtti
+      else:
+        result.baseKind = bcNoneRtti
+    elif m.compileToCpp:
+      result.baseKind = bcCppInherit
+    else:
+      result.baseKind = bcSupField
+  elif baseType.len != 0:
+    if m.compileToCpp:
+      result.baseKind = bcCppInherit
+    else:
+      result.baseKind = bcSupField
+  if result.baseKind == bcCppInherit:
+    obj.add(" : public ")
+    obj.add(baseType)
+  obj.add(" ")
+  obj.add("{\n")
+  result.preFieldsLen = obj.buf.len
+  case result.baseKind
+  of bcNone:
+    # rest of the options add a field or don't need it due to inheritance,
+    # we need to add the dummy field for uncheckedarray ahead of time
+    # so that it remains trailing
+    if t.itemId notin m.g.graph.memberProcsPerType and
+        t.n != nil and t.n.len == 1 and t.n[0].kind == nkSym and
+        t.n[0].sym.typ.skipTypes(abstractInst).kind == tyUncheckedArray:
+      # only consists of flexible array field, add *initial* dummy field
+      obj.addField(name = "dummy", typ = "char")
+  of bcCppInherit: discard
+  of bcNoneRtti:
+    obj.addField(name = "m_type", typ = ptrType(cgsymValue(m, "TNimType")))
+  of bcNoneTinyRtti:
+    obj.addField(name = "m_type", typ = ptrType(cgsymValue(m, "TNimTypeV2")))
+  of bcSupField:
+    obj.addField(name = "Sup", typ = baseType)
+
+proc finishStruct(obj: var Builder; m: BModule; t: PType; info: StructBuilderInfo) =
+  if info.baseKind == bcNone and info.preFieldsLen == obj.buf.len and
+      t.itemId notin m.g.graph.memberProcsPerType:
+    # no fields were added, add dummy field
+    obj.addField(name = "dummy", typ = "char")
+  if info.named:
+    obj.add("};\n")
+  else:
+    obj.add("}")
+  if tfPacked in t.flags and hasAttribute notin CC[m.config.cCompiler].props:
+    obj.add("#pragma pack(pop)\n")
+
+template addStruct(obj: var Builder; m: BModule; typ: PType; name: string; baseType: Snippet; body: typed) =
+  ## builds a struct type directly based on `typ` with fields according to `body`,
+  ## `name` can be empty to build as a type expression and not a statement
+  let info = startStruct(obj, m, typ, name, baseType)
+  body
+  finishStruct(obj, m, typ, info)
+
+template addFieldWithStructType(obj: var Builder; m: BModule; parentTyp: PType; fieldName: string, body: typed) =
+  ## adds a field with a `struct { ... }` type, building the fields according to `body`
+  obj.add('\t')
+  if tfPacked in parentTyp.flags:
+    if hasAttribute in CC[m.config.cCompiler].props:
+      obj.add("struct __attribute__((__packed__)) {\n")
+    else:
+      obj.add("#pragma pack(push, 1)\nstruct {")
+  else:
+    obj.add("struct {\n")
+  body
+  obj.add("} ")
+  obj.add(fieldName)
+  obj.add(";\n")
+  if tfPacked in parentTyp.flags and hasAttribute notin CC[m.config.cCompiler].props:
+    result.add("#pragma pack(pop)\n")
+
+template addAnonUnion(obj: var Builder; body: typed) =
+  ## adds an anonymous union i.e. `union { ... };` with fields according to `body`
+  obj.add "union{\n"
+  body
+  obj.add("};\n")
+
+type DeclVisibility = enum
+  None
+  Extern
+  ExternC
+  ImportLib
+  ExportLib
+  ExportLibVar
+  Private
+  StaticProc
+
+template addDeclWithVisibility(builder: var Builder, visibility: DeclVisibility, declBody: typed) =
+  ## adds a declaration as in `declBody` with the given visibility
+  case visibility
+  of None: discard
+  of Extern:
+    builder.add("extern ")
+  of ExternC:
+    builder.add("extern \"C\" ")
+  of ImportLib:
+    builder.add("N_LIB_IMPORT ")
+  of ExportLib:
+    builder.add("N_LIB_EXPORT ")
+  of ExportLibVar:
+    builder.add("N_LIB_EXPORT_VAR ")
+  of Private:
+    builder.add("N_LIB_PRIVATE ")
+  of StaticProc:
+    builder.add("static ")
+  declBody
+
+type ProcParamBuilder = object
+  needsComma: bool
+
+proc initProcParamBuilder(builder: var Builder): ProcParamBuilder =
+  result = ProcParamBuilder(needsComma: false)
+  builder.add("(")
+
+proc finishProcParamBuilder(builder: var Builder, params: ProcParamBuilder) =
+  if params.needsComma:
+    builder.add(")")
+  else:
+    builder.add("void)")
+
+template cgDeclFrmt*(s: PSym): string =
+  s.constraint.strVal
+
+proc addParam(builder: var Builder, params: var ProcParamBuilder, name: string, typ: Snippet) =
+  if params.needsComma:
+    builder.add(", ")
+  else:
+    params.needsComma = true
+  builder.add(typ)
+  builder.add(" ")
+  builder.add(name)
+
+proc addParam(builder: var Builder, params: var ProcParamBuilder, param: PSym, typ: Snippet) =
+  if params.needsComma:
+    builder.add(", ")
+  else:
+    params.needsComma = true
+  var modifiedTyp = typ
+  if sfNoalias in param.flags:
+    modifiedTyp.add(" NIM_NOALIAS")
+  if sfCodegenDecl notin param.flags:
+    builder.add(modifiedTyp)
+    builder.add(" ")
+    builder.add(param.loc.snippet)
+  else:
+    builder.add runtimeFormat(param.cgDeclFrmt, [modifiedTyp, param.loc.snippet])
+
+proc addUnnamedParam(builder: var Builder, params: var ProcParamBuilder, typ: Snippet) =
+  if params.needsComma:
+    builder.add(", ")
+  else:
+    params.needsComma = true
+  builder.add(typ)
+
+proc addVarargsParam(builder: var Builder, params: var ProcParamBuilder) =
+  # does not exist in NIFC, needs to be proc pragma
+  if params.needsComma:
+    builder.add(", ")
+  else:
+    params.needsComma = true
+  builder.add("...")
+
+template addProcParams(builder: var Builder, params: out ProcParamBuilder, body: typed) =
+  params = initProcParamBuilder(builder)
+  body
+  finishProcParamBuilder(builder, params)
+
+proc addProcHeader(builder: var Builder, m: BModule, prc: PSym, name: string, params, rettype: Snippet, addAttributes: bool) =
+  # on nifc should build something like (proc name params type pragmas
+  # with no body given
+  let noreturn = isNoReturn(m, prc)
+  if sfPure in prc.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
+    builder.add("__declspec(naked) ")
+  if noreturn and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
+    builder.add("__declspec(noreturn) ")
+  builder.add(CallingConvToStr[prc.typ.callConv])
+  builder.add("(")
+  builder.add(rettype)
+  builder.add(", ")
+  builder.add(name)
+  builder.add(")")
+  builder.add(params)
+  if addAttributes:
+    if sfPure in prc.flags and hasAttribute in extccomp.CC[m.config.cCompiler].props:
+      builder.add(" __attribute__((naked))")
+    if noreturn and hasAttribute in extccomp.CC[m.config.cCompiler].props:
+      builder.add(" __attribute__((noreturn))")
+
+proc finishProcHeaderAsProto(builder: var Builder) =
+  builder.add(";\n")
+
+template finishProcHeaderWithBody(builder: var Builder, body: typed) =
+  builder.add(" {\n")
+  body
+  builder.add("}\n\n")
+
+proc addProcVar(builder: var Builder, m: BModule, prc: PSym, name: string, params, rettype: Snippet,
+                isStatic = false, ignoreAttributes = false) =
+  # on nifc, builds full variable
+  if isStatic:
+    builder.add("static ")
+  let noreturn = isNoReturn(m, prc)
+  if not ignoreAttributes:
+    if sfPure in prc.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
+      builder.add("__declspec(naked) ")
+    if noreturn and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
+      builder.add("__declspec(noreturn) ")
+  builder.add(CallingConvToStr[prc.typ.callConv])
+  builder.add("_PTR(")
+  builder.add(rettype)
+  builder.add(", ")
+  builder.add(name)
+  builder.add(")")
+  builder.add(params)
+  if not ignoreAttributes:
+    if sfPure in prc.flags and hasAttribute in extccomp.CC[m.config.cCompiler].props:
+      builder.add(" __attribute__((naked))")
+    if noreturn and hasAttribute in extccomp.CC[m.config.cCompiler].props:
+      builder.add(" __attribute__((noreturn))")
+  # ensure we are just adding a variable:
+  builder.add(";\n")

compiler/cbuilderexprs.nim

@@ -0,0 +1,232 @@
+# XXX make complex ones like bitOr use builder instead
+# XXX add stuff like NI, NIM_NIL as constants
+
+proc constType(t: Snippet): Snippet =
+  # needs manipulation of `t` in nifc
+  "NIM_CONST " & t
+
+proc constPtrType(t: Snippet): Snippet =
+  t & "* NIM_CONST"
+
+proc ptrConstType(t: Snippet): Snippet =
+  "NIM_CONST " & t & "*"
+
+proc ptrType(t: Snippet): Snippet =
+  t & "*"
+
+const
+  CallingConvToStr: array[TCallingConvention, string] = ["N_NIMCALL",
+    "N_STDCALL", "N_CDECL", "N_SAFECALL",
+    "N_SYSCALL", # this is probably not correct for all platforms,
+                 # but one can #define it to what one wants
+    "N_INLINE", "N_NOINLINE", "N_FASTCALL", "N_THISCALL", "N_CLOSURE", "N_NOCONV",
+    "N_NOCONV" #ccMember is N_NOCONV
+    ]
+
+proc procPtrTypeUnnamed(rettype, params: Snippet): Snippet =
+  rettype & "(*)" & params
+
+proc procPtrTypeUnnamedNimCall(rettype, params: Snippet): Snippet =
+  rettype & "(N_RAW_NIMCALL*)" & params
+
+proc cCast(typ, value: Snippet): Snippet =
+  "((" & typ & ") " & value & ")"
+
+proc wrapPar(value: Snippet): Snippet =
+  # used for expression group, no-op on sexp
+  "(" & value & ")"
+
+proc removeSinglePar(value: Snippet): Snippet =
+  # removes a single paren layer expected to exist, to silence Wparentheses-equality
+  assert value[0] == '(' and value[^1] == ')'
+  value[1..^2]
+
+template addCast(builder: var Builder, typ: Snippet, valueBody: typed) =
+  ## adds a cast to `typ` with value built by `valueBody`
+  builder.add "(("
+  builder.add typ
+  builder.add ") "
+  valueBody
+  builder.add ")"
+
+proc cAddr(value: Snippet): Snippet =
+  "&" & value
+
+proc cDeref(value: Snippet): Snippet =
+  "(*" & value & ")"
+
+proc subscript(a, b: Snippet): Snippet =
+  a & "[" & b & "]"
+
+proc dotField(a, b: Snippet): Snippet =
+  a & "." & b
+
+proc derefField(a, b: Snippet): Snippet =
+  a & "->" & b
+
+proc bitOr(a, b: Snippet): Snippet =
+  "(" & a & " | " & b & ")"
+
+type CallBuilder = object
+  needsComma: bool
+
+proc initCallBuilder(builder: var Builder, callee: Snippet): CallBuilder =
+  result = CallBuilder(needsComma: false)
+  builder.add(callee)
+  builder.add("(")
+
+template addArgument(builder: var Builder, call: var CallBuilder, valueBody: typed) =
+  if call.needsComma:
+    builder.add(", ")
+  else:
+    call.needsComma = true
+  valueBody
+
+proc finishCallBuilder(builder: var Builder, call: CallBuilder) =
+  builder.add(")")
+
+template addCall(builder: var Builder, call: out CallBuilder, callee: Snippet, body: typed) =
+  call = initCallBuilder(builder, callee)
+  body
+  finishCallBuilder(builder, call)
+
+proc addCall(builder: var Builder, callee: Snippet, args: varargs[Snippet]) =
+  builder.add(callee)
+  builder.add("(")
+  if args.len != 0:
+    builder.add(args[0])
+    for i in 1 ..< args.len:
+      builder.add(", ")
+      builder.add(args[i])
+  builder.add(")")
+
+proc cCall(callee: Snippet, args: varargs[Snippet]): Snippet =
+  result = callee
+  result.add("(")
+  if args.len != 0:
+    result.add(args[0])
+    for i in 1 ..< args.len:
+      result.add(", ")
+      result.add(args[i])
+  result.add(")")
+
+proc addSizeof(builder: var Builder, val: Snippet) =
+  builder.add("sizeof(")
+  builder.add(val)
+  builder.add(")")
+
+proc addAlignof(builder: var Builder, val: Snippet) =
+  builder.add("NIM_ALIGNOF(")
+  builder.add(val)
+  builder.add(")")
+
+proc addOffsetof(builder: var Builder, val, member: Snippet) =
+  builder.add("offsetof(")
+  builder.add(val)
+  builder.add(", ")
+  builder.add(member)
+  builder.add(")")
+
+template cSizeof(val: Snippet): Snippet =
+  "sizeof(" & val & ")"
+
+template cAlignof(val: Snippet): Snippet =
+  "NIM_ALIGNOF(" & val & ")"
+
+template cOffsetof(val, member: Snippet): Snippet =
+  "offsetof(" & val & ", " & member & ")"
+
+type TypedBinaryOp = enum
+  Add, Sub, Mul, Div, Mod
+  Shr, Shl, BitAnd, BitOr, BitXor
+
+const typedBinaryOperators: array[TypedBinaryOp, string] = [
+  Add: "+",
+  Sub: "-",
+  Mul: "*",
+  Div: "/",
+  Mod: "%",
+  Shr: ">>",
+  Shl: "<<",
+  BitAnd: "&",
+  BitOr: "|",
+  BitXor: "^"
+]
+
+type TypedUnaryOp = enum
+  Neg, BitNot
+
+const typedUnaryOperators: array[TypedUnaryOp, string] = [
+  Neg: "-",
+  BitNot: "~",
+]
+
+type UntypedBinaryOp = enum
+  LessEqual, LessThan, GreaterEqual, GreaterThan, Equal, NotEqual
+  And, Or
+
+const untypedBinaryOperators: array[UntypedBinaryOp, string] = [
+  LessEqual: "<=",
+  LessThan: "<",
+  GreaterEqual: ">=",
+  GreaterThan: ">",
+  Equal: "==",
+  NotEqual: "!=",
+  And: "&&",
+  Or: "||"
+]
+
+type UntypedUnaryOp = enum
+  Not
+
+const untypedUnaryOperators: array[UntypedUnaryOp, string] = [
+  Not: "!"
+]
+
+proc addOp(builder: var Builder, binOp: TypedBinaryOp, t: Snippet, a, b: Snippet) =
+  builder.add('(')
+  builder.add(a)
+  builder.add(' ')
+  builder.add(typedBinaryOperators[binOp])
+  builder.add(' ')
+  builder.add(b)
+  builder.add(')')
+
+proc addOp(builder: var Builder, unOp: TypedUnaryOp, t: Snippet, a: Snippet) =
+  builder.add('(')
+  builder.add(typedUnaryOperators[unOp])
+  builder.add('(')
+  builder.add(a)
+  builder.add("))")
+
+proc addOp(builder: var Builder, binOp: UntypedBinaryOp, a, b: Snippet) =
+  builder.add('(')
+  builder.add(a)
+  builder.add(' ')
+  builder.add(untypedBinaryOperators[binOp])
+  builder.add(' ')
+  builder.add(b)
+  builder.add(')')
+
+proc addOp(builder: var Builder, unOp: UntypedUnaryOp, a: Snippet) =
+  builder.add('(')
+  builder.add(untypedUnaryOperators[unOp])
+  builder.add('(')
+  builder.add(a)
+  builder.add("))")
+
+template cOp(binOp: TypedBinaryOp, t: Snippet, a, b: Snippet): Snippet =
+  '(' & a & ' ' & typedBinaryOperators[binOp] & ' ' & b & ')'
+
+template cOp(binOp: TypedUnaryOp, t: Snippet, a: Snippet): Snippet =
+  '(' & typedUnaryOperators[binOp] & '(' & a & "))"
+
+template cOp(binOp: UntypedBinaryOp, a, b: Snippet): Snippet =
+  '(' & a & ' ' & untypedBinaryOperators[binOp] & ' ' & b & ')'
+
+template cOp(binOp: UntypedUnaryOp, a: Snippet): Snippet =
+  '(' & untypedUnaryOperators[binOp] & '(' & a & "))"
+
+template cIfExpr(cond, a, b: Snippet): Snippet =
+  # XXX used for `min` and `max`, maybe add nifc primitives for these
+  "(" & cond & " ? " & a & " : " & b & ")"

compiler/cbuilderstmts.nim

@@ -0,0 +1,150 @@
+template addAssignmentWithValue(builder: var Builder, lhs: Snippet, valueBody: typed) =
+  builder.add(lhs)
+  builder.add(" = ")
+  valueBody
+  builder.add(";\n")
+
+template addFieldAssignmentWithValue(builder: var Builder, lhs: Snippet, name: string, valueBody: typed) =
+  builder.add(lhs)
+  builder.add("." & name & " = ")
+  valueBody
+  builder.add(";\n")
+
+template addAssignment(builder: var Builder, lhs, rhs: Snippet) =
+  builder.addAssignmentWithValue(lhs):
+    builder.add(rhs)
+
+template addFieldAssignment(builder: var Builder, lhs: Snippet, name: string, rhs: Snippet) =
+  builder.addFieldAssignmentWithValue(lhs, name):
+    builder.add(rhs)
+
+template addMutualFieldAssignment(builder: var Builder, lhs, rhs: Snippet, name: string) =
+  builder.addFieldAssignmentWithValue(lhs, name):
+    builder.add(rhs)
+    builder.add("." & name)
+
+template addAssignment(builder: var Builder, lhs: Snippet, rhs: int | int64 | uint64 | Int128) =
+  builder.addAssignmentWithValue(lhs):
+    builder.addIntValue(rhs)
+
+template addFieldAssignment(builder: var Builder, lhs: Snippet, name: string, rhs: int | int64 | uint64 | Int128) =
+  builder.addFieldAssignmentWithValue(lhs, name):
+    builder.addIntValue(rhs)
+
+template addDerefFieldAssignment(builder: var Builder, lhs: Snippet, name: string, rhs: Snippet) =
+  builder.add(lhs)
+  builder.add("->" & name & " = ")
+  builder.add(rhs)
+  builder.add(";\n")
+
+template addSubscriptAssignment(builder: var Builder, lhs: Snippet, index: Snippet, rhs: Snippet) =
+  builder.add(lhs)
+  builder.add("[" & index & "] = ")
+  builder.add(rhs)
+  builder.add(";\n")
+
+template addStmt(builder: var Builder, stmtBody: typed) =
+  ## makes an expression built by `stmtBody` into a statement
+  stmtBody
+  builder.add(";\n")
+
+proc addCallStmt(builder: var Builder, callee: Snippet, args: varargs[Snippet]) =
+  builder.addStmt():
+    builder.addCall(callee, args)
+
+# XXX blocks need indent tracker in `Builder` object
+
+template addSingleIfStmt(builder: var Builder, cond: Snippet, body: typed) =
+  builder.add("if (")
+  builder.add(cond)
+  builder.add(") {\n")
+  body
+  builder.add("}\n")
+
+template addSingleIfStmtWithCond(builder: var Builder, condBody: typed, body: typed) =
+  builder.add("if (")
+  condBody
+  builder.add(") {\n")
+  body
+  builder.add("}\n")
+
+type IfStmt = object
+  needsElse: bool
+
+template addIfStmt(builder: var Builder, stmt: out IfStmt, body: typed) =
+  stmt = IfStmt(needsElse: false)
+  body
+  builder.add("\n")
+
+template addElifBranch(builder: var Builder, stmt: var IfStmt, cond: Snippet, body: typed) =
+  if stmt.needsElse:
+    builder.add(" else ")
+  else:
+    stmt.needsElse = true
+  builder.add("if (")
+  builder.add(cond)
+  builder.add(") {\n")
+  body
+  builder.add("}")
+
+template addElseBranch(builder: var Builder, stmt: var IfStmt, body: typed) =
+  assert stmt.needsElse
+  builder.add(" else {\n")
+  body
+  builder.add("}")
+
+proc addForRangeHeader(builder: var Builder, i, start, bound: Snippet, inclusive: bool = false) =
+  builder.add("for (")
+  builder.add(i)
+  builder.add(" = ")
+  builder.add(start)
+  builder.add("; ")
+  builder.add(i)
+  if inclusive:
+    builder.add(" <= ")
+  else:
+    builder.add(" < ")
+  builder.add(bound)
+  builder.add("; ")
+  builder.add(i)
+  builder.add("++) {\n")
+
+template addForRangeExclusive(builder: var Builder, i, start, bound: Snippet, body: typed) =
+  addForRangeHeader(builder, i, start, bound, false)
+  body
+  builder.add("}\n")
+
+template addForRangeInclusive(builder: var Builder, i, start, bound: Snippet, body: typed) =
+  addForRangeHeader(builder, i, start, bound, true)
+  body
+  builder.add("}\n")
+
+template addScope(builder: var Builder, body: typed) =
+  builder.add("{")
+  body
+  builder.add("\t}")
+
+proc addLabel(builder: var Builder, name: TLabel) =
+  builder.add(name)
+  builder.add(": ;\n")
+
+proc addReturn(builder: var Builder) =
+  builder.add("return;\n")
+
+proc addReturn(builder: var Builder, value: string) =
+  builder.add("return ")
+  builder.add(value)
+  builder.add(";\n")
+
+template addGoto(builder: var Builder, label: TLabel) =
+  builder.add("goto ")
+  builder.add(label)
+  builder.add(";\n")
+
+template addIncr(builder: var Builder, val: Snippet) =
+  builder.add(val)
+  builder.add("++;\n")
+
+template addDecr(builder: var Builder, val: Snippet) =
+  builder.add(val)
+  builder.add("--;\n")

compiler/ccgcalls.nim

@@ -76,6 +76,23 @@ proc isHarmlessStore(p: BProc; canRaise: bool; d: TLoc): bool =
   else:
     result = false
 
+proc cleanupTemp(p: BProc; returnType: PType, tmp: TLoc): bool =
+  if returnType.kind in {tyVar, tyLent}:
+    # we don't need to worry about var/lent return types
+    result = false
+  elif hasDestructor(returnType) and getAttachedOp(p.module.g.graph, returnType, attachedDestructor) != nil:
+    let dtor = getAttachedOp(p.module.g.graph, returnType, attachedDestructor)
+    var op = initLocExpr(p, newSymNode(dtor))
+    var callee = rdLoc(op)
+    let destroy = if dtor.typ.firstParamType.kind == tyVar:
+        callee & "(&" & rdLoc(tmp) & ")"
+      else:
+        callee & "(" & rdLoc(tmp) & ")"
+    raiseExitCleanup(p, destroy)
+    result = true
+  else:
+    result = false
+
 proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
@@ -115,7 +132,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
           # with them to prevent undefined behaviour and because the codegen
           # is free to emit expressions multiple times!
           d.k = locCall
-          d.r = pl
+          d.snippet = pl
           excl d.flags, lfSingleUse
         else:
           if d.k == locNone and p.splitDecls == 0:
@@ -123,35 +140,48 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
           else:
             if d.k == locNone: d = getTemp(p, typ.returnType)
             var list = initLoc(locCall, d.lode, OnUnknown)
-            list.r = pl
-            genAssignment(p, d, list, {}) # no need for deep copying
+            list.snippet = pl
+            genAssignment(p, d, list, {needAssignCall}) # no need for deep copying
             if canRaise: raiseExit(p)
 
       elif isHarmlessStore(p, canRaise, d):
-        if d.k == locNone: d = getTemp(p, typ.returnType)
+        var useTemp = false
+        if d.k == locNone:
+          useTemp = true
+          d = getTemp(p, typ.returnType)
         assert(d.t != nil)        # generate an assignment to d:
         var list = initLoc(locCall, d.lode, OnUnknown)
-        list.r = pl
-        genAssignment(p, d, list, flags) # no need for deep copying
-        if canRaise: raiseExit(p)
+        list.snippet = pl
+        genAssignment(p, d, list, flags+{needAssignCall}) # no need for deep copying
+        if canRaise:
+          if not (useTemp and cleanupTemp(p, typ.returnType, d)):
+            raiseExit(p)
       else:
         var tmp: TLoc = getTemp(p, typ.returnType, needsInit=true)
         var list = initLoc(locCall, d.lode, OnUnknown)
-        list.r = pl
-        genAssignment(p, tmp, list, flags) # no need for deep copying
-        if canRaise: raiseExit(p)
+        list.snippet = pl
+        genAssignment(p, tmp, list, flags+{needAssignCall}) # no need for deep copying
+        if canRaise:
+          if not cleanupTemp(p, typ.returnType, tmp):
+            raiseExit(p)
         genAssignment(p, d, tmp, {})
   else:
     pl.add(");\n")
     line(p, cpsStmts, pl)
     if canRaise: raiseExit(p)
 
-proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
+proc genBoundsCheck(p: BProc; arr, a, b: TLoc; arrTyp: PType)
 
 proc reifiedOpenArray(n: PNode): bool {.inline.} =
   var x = n
-  while x.kind in {nkAddr, nkHiddenAddr, nkHiddenStdConv, nkHiddenDeref}:
-    x = x[0]
+  while true:
+    case x.kind
+    of {nkAddr, nkHiddenAddr, nkHiddenDeref}:
+      x = x[0]
+    of nkHiddenStdConv:
+      x = x[1]
+    else:
+      break
   if x.kind == nkSym and x.sym.kind == skParam:
     result = false
   else:
@@ -161,12 +191,15 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF
   var a = initLocExpr(p, q[1])
   var b = initLocExpr(p, q[2])
   var c = initLocExpr(p, q[3])
+  # bug #23321: In the function mapType, ptrs (tyPtr, tyVar, tyLent, tyRef)
+  # are mapped into ctPtrToArray, the dereference of which is skipped
+  # in the `genDeref`. We need to skip these ptrs here
+  let ty = skipTypes(a.t, abstractVar+{tyPtr, tyRef})
   # but first produce the required index checks:
   if optBoundsCheck in p.options:
-    genBoundsCheck(p, a, b, c)
+    genBoundsCheck(p, a, b, c, ty)
   if prepareForMutation:
     linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
-  let ty = skipTypes(a.t, abstractVar+{tyPtr})
   let dest = getTypeDesc(p.module, destType)
   let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)]
   case ty.kind
@@ -176,8 +209,7 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF
       result = ("($3*)(($1)+($2))" % [rdLoc(a), rdLoc(b), dest],
                 lengthExpr)
     else:
-      var lit = newRopeAppender()
-      intLiteral(first, lit)
+      let lit = cIntLiteral(first)
       result = ("($4*)($1)+(($2)-($3))" %
         [rdLoc(a), rdLoc(b), lit, dest],
         lengthExpr)
@@ -241,7 +273,7 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode; result: var Rope) =
           optSeqDestructors in p.config.globalOptions:
         linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
       if ntyp.kind in {tyVar} and not compileToCpp(p.module):
-        var t = TLoc(r: "(*$1)" % [a.rdLoc])
+        var t = TLoc(snippet: "(*$1)" % [a.rdLoc])
         result.add "($4) ? ((*$1)$3) : NIM_NIL, $2" %
                      [a.rdLoc, lenExpr(p, t), dataField(p),
                       dataFieldAccessor(p, "*" & a.rdLoc)]
@@ -253,7 +285,7 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode; result: var Rope) =
     of tyPtr, tyRef:
       case elementType(a.t).kind
       of tyString, tySequence:
-        var t = TLoc(r: "(*$1)" % [a.rdLoc])
+        var t = TLoc(snippet: "(*$1)" % [a.rdLoc])
         result.add "($4) ? ((*$1)$3) : NIM_NIL, $2" %
                      [a.rdLoc, lenExpr(p, t), dataField(p),
                       dataFieldAccessor(p, "*" & a.rdLoc)]
@@ -298,18 +330,33 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Rope; need
       addAddrLoc(p.config, withTmpIfNeeded(p, a, needsTmp), result)
   elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
-    a = initLocExprSingleUse(p, n[0])
+    # bug #23748: we need to introduce a temporary here. The expression type
+    # will be a reference in C++ and we cannot create a temporary reference
+    # variable. Thus, we create a temporary pointer variable instead.
+    let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray
+    if needsIndirect:
+      n.typ() = n.typ.exactReplica
+      n.typ.flags.incl tfVarIsPtr
+    a = initLocExprSingleUse(p, n)
+    a = withTmpIfNeeded(p, a, needsTmp)
+    if needsIndirect: a.flags.incl lfIndirect
     # if the proc is 'importc'ed but not 'importcpp'ed then 'var T' still
     # means '*T'. See posix.nim for lots of examples that do that in the wild.
     let callee = call[0]
     if callee.kind == nkSym and
         {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
-        {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
+        {lfHeader, lfNoDecl} * callee.sym.loc.flags != {} and
+        needsIndirect:
       addAddrLoc(p.config, a, result)
     else:
       addRdLoc(a, result)
   else:
     a = initLocExprSingleUse(p, n)
+    if param.typ.kind in {tyVar, tyPtr, tyRef, tySink}:
+      let typ = skipTypes(param.typ, abstractPtrs)
+      if not sameBackendTypePickyAliases(typ, n.typ.skipTypes(abstractPtrs)):
+        a.snippet = "(($1) ($2))" %
+          [getTypeDesc(p.module, param.typ), rdCharLoc(a)]
     addRdLoc(withTmpIfNeeded(p, a, needsTmp), result)
   #assert result != nil
 
@@ -354,7 +401,7 @@ proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) =
   of nkCallKinds:
     case n.getMagic:
     of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
-        mAddr, mNew, mNewFinalize, mWasMoved, mDestroy, mReset:
+        mAddr, mNew, mNewFinalize, mWasMoved, mDestroy:
       getPotentialWrites(n[1], true, result)
       for i in 2..<n.len:
         getPotentialWrites(n[i], mutate, result)
@@ -392,9 +439,11 @@ proc genParams(p: BProc, ri: PNode, typ: PType; result: var Rope) =
         if not needTmp[i - 1]:
           needTmp[i - 1] = potentialAlias(n, potentialWrites)
       getPotentialWrites(ri[i], false, potentialWrites)
-    if ri[i].kind in {nkHiddenAddr, nkAddr}:
-      # Optimization: don't use a temp, if we would only take the address anyway
-      needTmp[i - 1] = false
+    when false:
+      # this optimization is wrong, see bug #23748
+      if ri[i].kind in {nkHiddenAddr, nkAddr}:
+        # Optimization: don't use a temp, if we would only take the address anyway
+        needTmp[i - 1] = false
 
   var oldLen = result.len
   for i in 1..<ri.len:
@@ -482,9 +531,9 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc = initLoc(locCall, d.lode, OnUnknown)
       if tfIterator in typ.flags:
-        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
+        list.snippet = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
       else:
-        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
+        list.snippet = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
       genAssignment(p, d, list, {}) # no need for deep copying
       if canRaise: raiseExit(p)
     else:
@@ -492,9 +541,9 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc = initLoc(locCall, d.lode, OnUnknown)
       if tfIterator in typ.flags:
-        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
+        list.snippet = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
       else:
-        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
+        list.snippet = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
       genAssignment(p, tmp, list, {})
       if canRaise: raiseExit(p)
       genAssignment(p, d, tmp, {})
@@ -677,7 +726,7 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
   # don't call '$' here for efficiency:
-  let pat = $ri[0].sym.loc.r
+  let pat = $ri[0].sym.loc.snippet
   internalAssert p.config, pat.len > 0
   if pat.contains({'#', '(', '@', '\''}):
     var pl = newRopeAppender()
@@ -690,13 +739,13 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
         # with them to prevent undefined behaviour and because the codegen
         # is free to emit expressions multiple times!
         d.k = locCall
-        d.r = pl
+        d.snippet = pl
         excl d.flags, lfSingleUse
       else:
         if d.k == locNone: d = getTemp(p, typ.returnType)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc = initLoc(locCall, d.lode, OnUnknown)
-        list.r = pl
+        list.snippet = pl
         genAssignment(p, d, list, {}) # no need for deep copying
     else:
       pl.add(";\n")
@@ -706,7 +755,7 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
     var argsCounter = 0
     if 1 < ri.len:
       genThisArg(p, ri, 1, typ, pl)
-    pl.add(op.r)
+    pl.add(op.snippet)
     var params = newRopeAppender()
     for i in 2..<ri.len:
       genOtherArg(p, ri, i, typ, params, argsCounter)
@@ -721,12 +770,12 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
   assert(typ.kind == tyProc)
 
   # don't call '$' here for efficiency:
-  let pat = $ri[0].sym.loc.r
+  let pat = $ri[0].sym.loc.snippet
   internalAssert p.config, pat.len > 0
   var start = 3
   if ' ' in pat:
     start = 1
-    pl.add(op.r)
+    pl.add(op.snippet)
     if ri.len > 1:
       pl.add(": ")
       genArg(p, ri[1], typ.n[1].sym, ri, pl)
@@ -735,7 +784,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     if ri.len > 1:
       genArg(p, ri[1], typ.n[1].sym, ri, pl)
       pl.add(" ")
-    pl.add(op.r)
+    pl.add(op.snippet)
     if ri.len > 2:
       pl.add(": ")
       genArg(p, ri[2], typ.n[2].sym, ri, pl)
@@ -770,7 +819,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
       if d.k == locNone: d = getTemp(p, typ.returnType)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc = initLoc(locCall, ri, OnUnknown)
-      list.r = pl
+      list.snippet = pl
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
     pl.add("];\n")

compiler/ccgliterals.nim

@@ -36,52 +36,84 @@ proc genStringLiteralDataOnlyV1(m: BModule, s: string; result: var Rope) =
   cgsym(m, "TGenericSeq")
   let tmp = getTempName(m)
   result.add tmp
-  m.s[cfsStrData].addf("STRING_LITERAL($1, $2, $3);$n",
-       [tmp, makeCString(s), rope(s.len)])
+  var res = newBuilder("")
+  res.addVarWithTypeAndInitializer(AlwaysConst, name = tmp):
+    res.addSimpleStruct(m, name = "", baseType = ""):
+      res.addField(name = "Sup", typ = "TGenericSeq")
+      res.addArrayField(name = "data", elementType = "NIM_CHAR", len = s.len + 1)
+  do:
+    var strInit: StructInitializer
+    res.addStructInitializer(strInit, kind = siOrderedStruct):
+      res.addField(strInit, name = "Sup"):
+        var seqInit: StructInitializer
+        res.addStructInitializer(seqInit, kind = siOrderedStruct):
+          res.addField(seqInit, name = "len"):
+            res.addIntValue(s.len)
+          res.addField(seqInit, name = "reserved"):
+            res.add(cCast("NI", bitOr(cCast("NU", rope(s.len)), "NIM_STRLIT_FLAG")))
+      res.addField(strInit, name = "data"):
+        res.add(makeCString(s))
+  m.s[cfsStrData].add(extract(res))
 
-proc genStringLiteralV1(m: BModule; n: PNode; result: var Rope) =
+proc genStringLiteralV1(m: BModule; n: PNode; result: var Builder) =
   if s.isNil:
-    appcg(m, result, "((#NimStringDesc*) NIM_NIL)", [])
+    result.add(cCast(ptrType(cgsymValue(m, "NimStringDesc")), "NIM_NIL"))
   else:
     let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
+    var name: string = ""
     if id == m.labels:
       # string literal not found in the cache:
-      appcg(m, result, "((#NimStringDesc*) &", [])
-      genStringLiteralDataOnlyV1(m, n.strVal, result)
-      result.add ")"
+      genStringLiteralDataOnlyV1(m, n.strVal, name)
     else:
-      appcg(m, result, "((#NimStringDesc*) &$1$2)",
-                      [m.tmpBase, id])
+      name = m.tmpBase & $id
+    result.add(cCast(ptrType(cgsymValue(m, "NimStringDesc")), cAddr(name)))
 
 # ------ Version 2: destructor based strings and seqs -----------------------
 
 proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope; isConst: bool) =
-  m.s[cfsStrData].addf("static $4 struct {$n" &
-       "  NI cap; NIM_CHAR data[$2+1];$n" &
-       "} $1 = { $2 | NIM_STRLIT_FLAG, $3 };$n",
-       [result, rope(s.len), makeCString(s),
-       rope(if isConst: "const" else: "")])
+  var res = newBuilder("")
+  res.addVarWithTypeAndInitializer(
+      if isConst: AlwaysConst else: Global,
+      name = result):
+    res.addSimpleStruct(m, name = "", baseType = ""):
+      res.addField(name = "cap", typ = "NI")
+      res.addArrayField(name = "data", elementType = "NIM_CHAR", len = s.len + 1)
+  do:
+    var structInit: StructInitializer
+    res.addStructInitializer(structInit, kind = siOrderedStruct):
+      res.addField(structInit, name = "cap"):
+        res.add(bitOr(rope(s.len), "NIM_STRLIT_FLAG"))
+      res.addField(structInit, name = "data"):
+        res.add(makeCString(s))
+  m.s[cfsStrData].add(extract(res))
 
-proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool; result: var Rope) =
+proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool; result: var Builder) =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
+  var litName: string
   if id == m.labels:
-    let pureLit = getTempName(m)
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
-    let tmp = getTempName(m)
-    result.add tmp
     cgsym(m, "NimStrPayload")
     cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    m.s[cfsStrData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [tmp, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
+    litName = getTempName(m)
+    genStringLiteralDataOnlyV2(m, n.strVal, litName, isConst)
   else:
-    let tmp = getTempName(m)
-    result.add tmp
-    m.s[cfsStrData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [tmp, rope(n.strVal.len), m.tmpBase & rope(id),
-          rope(if isConst: "const" else: "")])
+    litName = m.tmpBase & $id
+  let tmp = getTempName(m)
+  result.add tmp
+  var res = newBuilder("")
+  res.addVarWithInitializer(
+      if isConst: AlwaysConst else: Global,
+      name = tmp,
+      typ = "NimStringV2"):
+    var strInit: StructInitializer
+    res.addStructInitializer(strInit, kind = siOrderedStruct):
+      res.addField(strInit, name = "len"):
+        res.addIntValue(n.strVal.len)
+      res.addField(strInit, name = "p"):
+        res.add(cCast(ptrType("NimStrPayload"), cAddr(litName)))
+  m.s[cfsStrData].add(extract(res))
 
-proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool; result: var Rope) =
+proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool; result: var Builder) =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   var pureLit: Rope
   if id == m.labels:
@@ -92,7 +124,12 @@ proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool; result: var Ro
     genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
   else:
     pureLit = m.tmpBase & rope(id)
-  result.addf "{$1, (NimStrPayload*)&$2}", [rope(n.strVal.len), pureLit]
+  var strInit: StructInitializer
+  result.addStructInitializer(strInit, kind = siOrderedStruct):
+    result.addField(strInit, name = "len"):
+      result.addIntValue(n.strVal.len)
+    result.addField(strInit, name = "p"):
+      result.add(cCast(ptrType("NimStrPayload"), cAddr(pureLit)))
 
 # ------ Version selector ---------------------------------------------------
 
@@ -107,10 +144,10 @@ proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
   else:
     localError(m.config, info, "cannot determine how to produce code for string literal")
 
-proc genNilStringLiteral(m: BModule; info: TLineInfo; result: var Rope) =
-  appcg(m, result, "((#NimStringDesc*) NIM_NIL)", [])
+proc genNilStringLiteral(m: BModule; info: TLineInfo; result: var Builder) =
+  result.add(cCast(ptrType(cgsymValue(m, "NimStringDesc")), "NIM_NIL"))
 
-proc genStringLiteral(m: BModule; n: PNode; result: var Rope) =
+proc genStringLiteral(m: BModule; n: PNode; result: var Builder) =
   case detectStrVersion(m)
   of 0, 1: genStringLiteralV1(m, n, result)
   of 2: genStringLiteralV2(m, n, isConst = true, result)

compiler/ccgreset.nim

@@ -24,10 +24,10 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
   of nkRecCase:
     if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
     let disc = n[0].sym
-    if disc.loc.r == "": fillObjectFields(p.module, typ)
+    if disc.loc.snippet == "": fillObjectFields(p.module, typ)
     if disc.loc.t == nil:
       internalError(p.config, n.info, "specializeResetN()")
-    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
+    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.snippet])
     for i in 1..<n.len:
       let branch = n[i]
       assert branch.kind in {nkOfBranch, nkElse}
@@ -38,14 +38,14 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
       specializeResetN(p, accessor, lastSon(branch), typ)
       lineF(p, cpsStmts, "break;$n", [])
     lineF(p, cpsStmts, "} $n", [])
-    specializeResetT(p, "$1.$2" % [accessor, disc.loc.r], disc.loc.t)
+    specializeResetT(p, "$1.$2" % [accessor, disc.loc.snippet], disc.loc.t)
   of nkSym:
     let field = n.sym
     if field.typ.kind == tyVoid: return
-    if field.loc.r == "": fillObjectFields(p.module, typ)
+    if field.loc.snippet == "": fillObjectFields(p.module, typ)
     if field.loc.t == nil:
       internalError(p.config, n.info, "specializeResetN()")
-    specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
+    specializeResetT(p, "$1.$2" % [accessor, field.loc.snippet], field.loc.t)
   else: internalError(p.config, n.info, "specializeResetN()")
 
 proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
@@ -59,8 +59,8 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
     let arraySize = lengthOrd(p.config, typ.indexType)
     var i: TLoc = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt))
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
-            [i.r, arraySize])
-    specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ.elementType)
+            [i.snippet, arraySize])
+    specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.snippet]), typ.elementType)
     lineF(p, cpsStmts, "}$n", [])
   of tyObject:
     var x = typ.baseClass
@@ -96,7 +96,7 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
       raiseAssert "unexpected set type kind"
   of tyNone, tyEmpty, tyNil, tyUntyped, tyTyped, tyGenericInvocation,
       tyGenericParam, tyOrdinal, tyOpenArray, tyForward, tyVarargs,
-      tyUncheckedArray, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
+      tyUncheckedArray, tyError, tyBuiltInTypeClass, tyUserTypeClass,
       tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot,
       tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable:
     discard

compiler/ccgstmts.nim

@@ -110,10 +110,10 @@ proc genVarTuple(p: BProc, n: PNode) =
       initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[^1]))
     var field = initLoc(locExpr, vn, tup.storage)
     if t.kind == tyTuple:
-      field.r = "$1.Field$2" % [rdLoc(tup), rope(i)]
+      field.snippet = "$1.Field$2" % [rdLoc(tup), rope(i)]
     else:
       if t.n[i].kind != nkSym: internalError(p.config, n.info, "genVarTuple")
-      field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n[i].sym)]
+      field.snippet = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n[i].sym)]
     putLocIntoDest(p, v.loc, field)
     if forHcr or isGlobalInBlock:
       hcrGlobals.add((loc: v.loc, tp: "NULL"))
@@ -128,7 +128,7 @@ proc genVarTuple(p: BProc, n: PNode) =
     lineCg(p, cpsLocals, "NIM_BOOL $1 = NIM_FALSE;$n", [hcrCond])
     for curr in hcrGlobals:
       lineCg(p, cpsLocals, "$1 |= hcrRegisterGlobal($4, \"$2\", sizeof($3), $5, (void**)&$2);$N",
-              [hcrCond, curr.loc.r, rdLoc(curr.loc), getModuleDllPath(p.module, n[0].sym), curr.tp])
+              [hcrCond, curr.loc.snippet, rdLoc(curr.loc), getModuleDllPath(p.module, n[0].sym), curr.tp])
 
 
 proc loadInto(p: BProc, le, ri: PNode, a: var TLoc) {.inline.} =
@@ -146,16 +146,16 @@ proc loadInto(p: BProc, le, ri: PNode, a: var TLoc) {.inline.} =
     a.flags.incl(lfEnforceDeref)
     expr(p, ri, a)
 
-proc assignLabel(b: var TBlock; result: var Rope) {.inline.} =
+proc assignLabel(b: var TBlock; result: var Builder) {.inline.} =
   b.label = "LA" & b.id.rope
   result.add b.label
 
-proc blockBody(b: var TBlock; result: var Rope) =
-  result.add b.sections[cpsLocals]
+proc blockBody(b: var TBlock; result: var Builder) =
+  result.add extract(b.sections[cpsLocals])
   if b.frameLen > 0:
     result.addf("FR_.len+=$1;$n", [b.frameLen.rope])
-  result.add(b.sections[cpsInit])
-  result.add(b.sections[cpsStmts])
+  result.add(extract(b.sections[cpsInit]))
+  result.add(extract(b.sections[cpsStmts]))
 
 proc endBlock(p: BProc, blockEnd: Rope) =
   let topBlock = p.blocks.len-1
@@ -267,11 +267,11 @@ proc genBreakState(p: BProc, n: PNode, d: var TLoc) =
 
   if n[0].kind == nkClosure:
     a = initLocExpr(p, n[0][1])
-    d.r = "(((NI*) $1)[1] < 0)" % [rdLoc(a)]
+    d.snippet = "(((NI*) $1)[1] < 0)" % [rdLoc(a)]
   else:
     a = initLocExpr(p, n[0])
     # the environment is guaranteed to contain the 'state' field at offset 1:
-    d.r = "((((NI*) $1.ClE_0)[1]) < 0)" % [rdLoc(a)]
+    d.snippet = "((((NI*) $1.ClE_0)[1]) < 0)" % [rdLoc(a)]
 
 proc genGotoVar(p: BProc; value: PNode) =
   if value.kind notin {nkCharLit..nkUInt64Lit}:
@@ -279,9 +279,9 @@ proc genGotoVar(p: BProc; value: PNode) =
   else:
     lineF(p, cpsStmts, "goto NIMSTATE_$#;$n", [value.intVal.rope])
 
-proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; result: var Rope)
+proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; result: var Builder)
 
-proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Rope) =
+proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Builder) =
   if lfDynamicLib in v.loc.flags or sfThread in v.flags or p.hcrOn:
     discard "nothing to do"
   elif sfGlobal in v.flags and value != nil and isDeepConstExpr(value, p.module.compileToCpp) and
@@ -289,7 +289,7 @@ proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Rope) =
     #echo "New code produced for ", v.name.s, " ", p.config $ value.info
     genBracedInit(p, value, isConst = false, v.typ, result)
 
-proc genCppParamsForCtor(p: BProc; call: PNode): string =
+proc genCppParamsForCtor(p: BProc; call: PNode; didGenTemp: var bool): string =
   result = ""
   var argsCounter = 0
   let typ = skipTypes(call[0].typ, abstractInst)
@@ -298,12 +298,23 @@ proc genCppParamsForCtor(p: BProc; call: PNode): string =
     #if it's a type we can just generate here another initializer as we are in an initializer context
     if call[i].kind == nkCall and call[i][0].kind == nkSym and call[i][0].sym.kind == skType:
       if argsCounter > 0: result.add ","
-      result.add genCppInitializer(p.module, p, call[i][0].sym.typ)
+      result.add genCppInitializer(p.module, p, call[i][0].sym.typ, didGenTemp)
     else:
+      #We need to test for temp in globals, see: #23657
+      let param =
+        if typ[i].kind in {tyVar} and call[i].kind == nkHiddenAddr:
+          call[i][0]
+        else:
+          call[i]
+      if param.kind != nkBracketExpr or param.typ.kind in
+        {tyRef, tyPtr, tyUncheckedArray, tyArray, tyOpenArray,
+          tyVarargs, tySequence, tyString, tyCstring, tyTuple}:
+        let tempLoc = initLocExprSingleUse(p, param)
+        didGenTemp = didGenTemp or tempLoc.k == locTemp
       genOtherArg(p, call, i, typ, result, argsCounter)
 
-proc genCppVarForCtor(p: BProc; call: PNode; decl: var Rope) =
-  let params = genCppParamsForCtor(p, call)
+proc genCppVarForCtor(p: BProc; call: PNode; decl: var Rope, didGenTemp: var bool) =
+  let params = genCppParamsForCtor(p, call, didGenTemp)
   if params.len == 0:
     decl = runtimeFormat("$#;\n", [decl])
   else:
@@ -319,8 +330,9 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     value.kind in nkCallKinds and value[0].kind == nkSym and
     v.typ.kind != tyPtr and sfConstructor in value[0].sym.flags
   var targetProc = p
-  var valueAsRope = ""
-  potentialValueInit(p, v, value, valueAsRope)
+  var valueBuilder = newBuilder("")
+  potentialValueInit(p, v, value, valueBuilder)
+  let valueAsRope = extract(valueBuilder)
   if sfGlobal in v.flags:
     if v.flags * {sfImportc, sfExportc} == {sfImportc} and
         value.kind == nkEmpty and
@@ -330,7 +342,14 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
       # v.owner.kind != skModule:
       targetProc = p.module.preInitProc
     if isCppCtorCall and not containsHiddenPointer(v.typ):
-      callGlobalVarCppCtor(targetProc, v, vn, value)
+      var didGenTemp = false
+      callGlobalVarCppCtor(targetProc, v, vn, value, didGenTemp)
+      if didGenTemp:
+        message(p.config, vn.info, warnGlobalVarConstructorTemporary, vn.sym.name.s)
+        #We fail to call the constructor in the global scope so we do the call inside the main proc
+        assignGlobalVar(targetProc, vn, valueAsRope)
+        var loc = initLocExprSingleUse(targetProc, value)
+        genAssignment(targetProc, v.loc, loc, {})
     else:
       assignGlobalVar(targetProc, vn, valueAsRope)
 
@@ -365,7 +384,8 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
       var decl = localVarDecl(p, vn)
       var tmp: TLoc
       if isCppCtorCall:
-        genCppVarForCtor(p, value, decl)
+        var didGenTemp = false
+        genCppVarForCtor(p, value, decl, didGenTemp)
         line(p, cpsStmts, decl)
       else:
         tmp = initLocExprSingleUse(p, value)
@@ -386,7 +406,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     # put it in the locals section - mainly because of loops which
     # use the var in a call to resetLoc() in the statements section
     lineCg(targetProc, cpsLocals, "hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1);$n",
-           [v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc])
+           [v.loc.snippet, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc])
     # nothing special left to do later on - let's avoid closing and reopening blocks
     forHcr = false
 
@@ -395,7 +415,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
   # be able to re-run it but without the top level code - just the init of globals
   if forHcr:
     lineCg(targetProc, cpsStmts, "if (hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1))$N",
-           [v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc])
+           [v.loc.snippet, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc])
     startBlock(targetProc)
   if value.kind != nkEmpty and valueAsRope.len == 0:
     genLineDir(targetProc, vn)
@@ -575,8 +595,7 @@ proc genComputedGoto(p: BProc; n: PNode) =
         return
 
       let val = getOrdValue(it[j])
-      var lit = newRopeAppender()
-      intLiteral(toInt64(val)+id+1, lit)
+      let lit = cIntLiteral(toInt64(val)+id+1)
       lineF(p, cpsStmts, "TMP$#_:$n", [lit])
 
     genStmts(p, it.lastSon)
@@ -736,6 +755,18 @@ proc raiseExit(p: BProc) =
       lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
         [p.nestedTryStmts[^1].label])
 
+proc raiseExitCleanup(p: BProc, destroy: string) =
+  assert p.config.exc == excGoto
+  if nimErrorFlagDisabled notin p.flags:
+    p.flags.incl nimErrorFlagAccessed
+    if p.nestedTryStmts.len == 0:
+      p.flags.incl beforeRetNeeded
+      # easy case, simply goto 'ret':
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) {$1; goto BeforeRet_;}$n", [destroy])
+    else:
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) {$2; goto LA$1_;}$n",
+        [p.nestedTryStmts[^1].label, destroy])
+
 proc finallyActions(p: BProc) =
   if p.config.exc != excGoto and p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept:
     # if the current try stmt have a finally block,
@@ -744,7 +775,7 @@ proc finallyActions(p: BProc) =
     if finallyBlock != nil:
       genSimpleBlock(p, finallyBlock[0])
 
-proc raiseInstr(p: BProc; result: var Rope) =
+proc raiseInstr(p: BProc; result: var Builder) =
   if p.config.exc == excGoto:
     let L = p.nestedTryStmts.len
     if L == 0:
@@ -765,10 +796,14 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     var e = rdLoc(a)
     discard getTypeDesc(p.module, t[0].typ)
     var typ = skipTypes(t[0].typ, abstractPtrs)
-    # XXX For reasons that currently escape me, this is only required by the new
-    # C++ based exception handling:
-    if p.config.exc == excCpp:
+    case p.config.exc
+    of excCpp:
       blockLeaveActions(p, howManyTrys = 0, howManyExcepts = p.inExceptBlockLen)
+    of excGoto:
+      blockLeaveActions(p, howManyTrys = 0,
+        howManyExcepts = (if p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept: 1 else: 0))
+    else:
+      discard
     genLineDir(p, t)
     if isImportedException(typ, p.config):
       lineF(p, cpsStmts, "throw $1;$n", [e])
@@ -890,8 +925,7 @@ proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) =
               [rdLoc(a), bitMask])
     for j in 0..high(branches):
       if branches[j] != "":
-        var lit = newRopeAppender()
-        intLiteral(j, lit)
+        let lit = cIntLiteral(j)
         lineF(p, cpsStmts, "case $1: $n$2break;$n",
              [lit, branches[j]])
     lineF(p, cpsStmts, "}$n", []) # else statement:
@@ -929,22 +963,22 @@ proc genCaseRange(p: BProc, branch: PNode) =
   for j in 0..<branch.len-1:
     if branch[j].kind == nkRange:
       if hasSwitchRange in CC[p.config.cCompiler].props:
-        var litA = newRopeAppender()
-        var litB = newRopeAppender()
+        var litA = newBuilder("")
+        var litB = newBuilder("")
         genLiteral(p, branch[j][0], litA)
         genLiteral(p, branch[j][1], litB)
-        lineF(p, cpsStmts, "case $1 ... $2:$n", [litA, litB])
+        lineF(p, cpsStmts, "case $1 ... $2:$n", [extract(litA), extract(litB)])
       else:
         var v = copyNode(branch[j][0])
         while v.intVal <= branch[j][1].intVal:
-          var litA = newRopeAppender()
+          var litA = newBuilder("")
           genLiteral(p, v, litA)
-          lineF(p, cpsStmts, "case $1:$n", [litA])
+          lineF(p, cpsStmts, "case $1:$n", [extract(litA)])
           inc(v.intVal)
     else:
-      var litA = newRopeAppender()
+      var litA = newBuilder("")
       genLiteral(p, branch[j], litA)
-      lineF(p, cpsStmts, "case $1:$n", [litA])
+      lineF(p, cpsStmts, "case $1:$n", [extract(litA)])
 
 proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
   # analyse 'case' statement:
@@ -1510,7 +1544,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false; result: var Rope) =
       else:
         discard getTypeDesc(p.module, skipTypes(sym.typ, abstractPtrs))
         fillBackendName(p.module, sym)
-        res.add($sym.loc.r)
+        res.add($sym.loc.snippet)
     of nkTypeOfExpr:
       res.add($getTypeDesc(p.module, it.typ))
     else:
@@ -1547,14 +1581,14 @@ proc genAsmStmt(p: BProc, t: PNode) =
       if whichPragma(i) == wAsmSyntax:
         asmSyntax = i[1].strVal
 
-  if asmSyntax != "" and 
+  if asmSyntax != "" and
      not (
       asmSyntax == "gcc" and hasGnuAsm in CC[p.config.cCompiler].props or
       asmSyntax == "vcc" and hasGnuAsm notin CC[p.config.cCompiler].props):
     localError(
-      p.config, t.info, 
+      p.config, t.info,
       "Your compiler does not support the specified inline assembler")
-  
+
   genAsmOrEmitStmt(p, t, isAsmStmt=true, s)
   # see bug #2362, "top level asm statements" seem to be a mis-feature
   # but even if we don't do this, the example in #2362 cannot possibly
@@ -1592,9 +1626,9 @@ proc genPragma(p: BProc, n: PNode) =
     case whichPragma(it)
     of wEmit: genEmit(p, it)
     of wPush:
-      processPushBackendOption(p.optionsStack, p.options, n, i+1)
+      processPushBackendOption(p.config, p.optionsStack, p.options, n, i+1)
     of wPop:
-      processPopBackendOption(p.optionsStack, p.options)
+      processPopBackendOption(p.config, p.optionsStack, p.options)
     else: discard
 
 
@@ -1606,8 +1640,7 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
   if not containsOrIncl(p.module.declaredThings, field.id):
     appcg(p.module, cfsVars, "extern $1",
           [discriminatorTableDecl(p.module, t, field)])
-  var lit = newRopeAppender()
-  intLiteral(toInt64(lengthOrd(p.config, field.typ))+1, lit)
+  let lit = cIntLiteral(toInt64(lengthOrd(p.config, field.typ))+1)
   lineCg(p, cpsStmts,
         "#FieldDiscriminantCheck((NI)(NU)($1), (NI)(NU)($2), $3, $4);$n",
         [rdLoc(a), rdLoc(tmp), discriminatorTableName(p.module, t, field),

compiler/ccgthreadvars.nim

@@ -30,7 +30,7 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
     # allocator for it :-(
     if not containsOrIncl(m.g.nimtvDeclared, s.id):
       m.g.nimtvDeps.add(s.loc.t)
-      m.g.nimtv.addf("$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
+      m.g.nimtv.addf("$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.snippet])
   else:
     if isExtern: m.s[cfsVars].add("extern ")
     elif lfExportLib in s.loc.flags: m.s[cfsVars].add("N_LIB_EXPORT_VAR ")
@@ -41,13 +41,15 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
         m.s[cfsVars].add("NIM_THREAD_LOCAL ")
       else: m.s[cfsVars].add("NIM_THREADVAR ")
     m.s[cfsVars].add(getTypeDesc(m, s.loc.t))
-    m.s[cfsVars].addf(" $1;$n", [s.loc.r])
+    m.s[cfsVars].addf(" $1;$n", [s.loc.snippet])
 
 proc generateThreadLocalStorage(m: BModule) =
   if m.g.nimtv != "" and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
     for t in items(m.g.nimtvDeps): discard getTypeDesc(m, t)
     finishTypeDescriptions(m)
-    m.s[cfsSeqTypes].addf("typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
+    m.s[cfsSeqTypes].addTypedef(name = "NimThreadVars"):
+      m.s[cfsSeqTypes].addSimpleStruct(m, name = "", baseType = ""):
+        m.s[cfsSeqTypes].add(m.g.nimtv)
 
 proc generateThreadVarsSize(m: BModule) =
   if m.g.nimtv != "":

compiler/ccgtrav.nim

@@ -34,10 +34,10 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
     if (n[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
     var p = c.p
     let disc = n[0].sym
-    if disc.loc.r == "": fillObjectFields(c.p.module, typ)
+    if disc.loc.snippet == "": fillObjectFields(c.p.module, typ)
     if disc.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
-    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
+    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.snippet])
     for i in 1..<n.len:
       let branch = n[i]
       assert branch.kind in {nkOfBranch, nkElse}
@@ -51,10 +51,10 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
   of nkSym:
     let field = n.sym
     if field.typ.kind == tyVoid: return
-    if field.loc.r == "": fillObjectFields(c.p.module, typ)
+    if field.loc.snippet == "": fillObjectFields(c.p.module, typ)
     if field.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
-    genTraverseProc(c, "$1.$2" % [accessor, field.loc.r], field.loc.t)
+    genTraverseProc(c, "$1.$2" % [accessor, field.loc.snippet], field.loc.t)
   else: internalError(c.p.config, n.info, "genTraverseProc()")
 
 proc parentObj(accessor: Rope; m: BModule): Rope {.inline.} =
@@ -76,12 +76,11 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
     let arraySize = lengthOrd(c.p.config, typ.indexType)
     var i: TLoc = getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt))
     var oldCode = p.s(cpsStmts)
-    freeze oldCode
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
-            [i.r, arraySize])
-    let oldLen = p.s(cpsStmts).len
-    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ.elementType)
-    if p.s(cpsStmts).len == oldLen:
+            [i.snippet, arraySize])
+    let oldLen = p.s(cpsStmts).buf.len
+    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.snippet]), typ.elementType)
+    if p.s(cpsStmts).buf.len == oldLen:
       # do not emit dummy long loops for faster debug builds:
       p.s(cpsStmts) = oldCode
     else:
@@ -119,14 +118,13 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   assert typ.kind == tySequence
   var i = getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt))
   var oldCode = p.s(cpsStmts)
-  freeze oldCode
-  var a = TLoc(r: accessor)
+  var a = TLoc(snippet: accessor)
 
   lineF(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
-      [i.r, lenExpr(c.p, a)])
-  let oldLen = p.s(cpsStmts).len
-  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ.elementType)
-  if p.s(cpsStmts).len == oldLen:
+      [i.snippet, lenExpr(c.p, a)])
+  let oldLen = p.s(cpsStmts).buf.len
+  genTraverseProc(c, "$1$3[$2]" % [accessor, i.snippet, dataField(c.p)], typ.elementType)
+  if p.s(cpsStmts).buf.len == oldLen:
     # do not emit dummy long loops for faster debug builds:
     p.s(cpsStmts) = oldCode
   else:
@@ -160,7 +158,7 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
       genTraverseProc(c, "(*a)".rope, typ.elementType)
 
   let generatedProc = "$1 {$n$2$3$4}\n" %
-        [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)]
+        [header, extract(p.s(cpsLocals)), extract(p.s(cpsInit)), extract(p.s(cpsStmts))]
 
   m.s[cfsProcHeaders].addf("$1;\n", [header])
   m.s[cfsProcs].add(generatedProc)
@@ -189,7 +187,7 @@ proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =
   genTraverseProc(c, sLoc, s.loc.t)
 
   let generatedProc = "$1 {$n$2$3$4}$n" %
-        [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)]
+        [header, extract(p.s(cpsLocals)), extract(p.s(cpsInit)), extract(p.s(cpsStmts))]
 
   m.s[cfsProcHeaders].addf("$1;$n", [header])
   m.s[cfsProcs].add(generatedProc)

compiler/ccgutils.nim

@@ -90,6 +90,9 @@ proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool =
     if s.typ.sym != nil and sfForward in s.typ.sym.flags:
       # forwarded objects are *always* passed by pointers for consistency!
       result = true
+    elif s.typ.kind == tySink and conf.selectedGC notin {gcArc, gcAtomicArc, gcOrc, gcHooks}:
+      # bug #23354:
+      result = false
     elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
       result = true           # requested anyway
     elif (tfFinal in pt.flags) and (pt[0] == nil):
@@ -122,7 +125,7 @@ proc encodeSym*(m: BModule; s: PSym; makeUnique: bool = false): string =
   var name = s.name.s
   if makeUnique:
     name = makeUnique(m, s, name)
-  "N" & encodeName(s.owner.name.s) & encodeName(name) & "E"
+  "N" & encodeName(s.skipGenericOwner.name.s) & encodeName(name) & "E"
 
 proc encodeType*(m: BModule; t: PType): string =
   result = ""

compiler/cgendata.nim

@@ -11,7 +11,7 @@
 
 import
   ast, ropes, options,
-  ndi, lineinfos, pathutils, modulegraphs
+  lineinfos, pathutils, modulegraphs, cbuilderbase
 
 import std/[intsets, tables, sets]
 
@@ -43,12 +43,12 @@ type
     ctUInt, ctUInt8, ctUInt16, ctUInt32, ctUInt64,
     ctArray, ctPtrToArray, ctStruct, ctPtr, ctNimStr, ctNimSeq, ctProc,
     ctCString
-  TCFileSections* = array[TCFileSection, Rope] # represents a generated C file
+  TCFileSections* = array[TCFileSection, Builder] # represents a generated C file
   TCProcSection* = enum       # the sections a generated C proc consists of
     cpsLocals,                # section of local variables for C proc
     cpsInit,                  # section for init of variables for C proc
     cpsStmts                  # section of local statements for C proc
-  TCProcSections* = array[TCProcSection, Rope] # represents a generated C proc
+  TCProcSections* = array[TCProcSection, Builder] # represents a generated C proc
   BModule* = ref TCGen
   BProc* = ref TCProc
   TBlock* = object
@@ -88,7 +88,7 @@ type
     options*: TOptions        # options that should be used for code
                               # generation; this is the same as prc.options
                               # unless prc == nil
-    optionsStack*: seq[TOptions]
+    optionsStack*: seq[(TOptions, TNoteKinds)]
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
     splitDecls*: int          # > 0 if we are in some context for C++ that
@@ -161,7 +161,7 @@ type
     typeInfoMarkerV2*: TypeCache
     initProc*: BProc          # code for init procedure
     preInitProc*: BProc       # code executed before the init proc
-    hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on
+    hcrCreateTypeInfosProc*: Builder # type info globals are in here when HCR=on
     inHcrInitGuard*: bool     # We are currently within a HCR reloading guard.
     typeStack*: TTypeSeq      # used for type generation
     dataCache*: TNodeTable
@@ -172,7 +172,6 @@ type
                                              # OpenGL wrapper
     sigConflicts*: CountTable[SigHash]
     g*: BModuleList
-    ndi*: NdiFile
 
 template config*(m: BModule): ConfigRef = m.g.config
 template config*(p: BProc): ConfigRef = p.module.g.config
@@ -182,18 +181,18 @@ proc includeHeader*(this: BModule; header: string) =
   if not this.headerFiles.contains header:
     this.headerFiles.add header
 
-proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
+proc s*(p: BProc, s: TCProcSection): var Builder {.inline.} =
   # section in the current block
   result = p.blocks[^1].sections[s]
 
-proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
+proc procSec*(p: BProc, s: TCProcSection): var Builder {.inline.} =
   # top level proc sections
   result = p.blocks[0].sections[s]
 
 proc initBlock*(): TBlock =
   result = TBlock()
   for i in low(result.sections)..high(result.sections):
-    result.sections[i] = newRopeAppender()
+    result.sections[i] = newBuilder("")
 
 proc newProc*(prc: PSym, module: BModule): BProc =
   result = BProc(

compiler/cgmeth.nim

@@ -55,7 +55,7 @@ proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   # replace ordinary method by dispatcher method:
   let disp = getDispatcher(result[0].sym)
   if disp != nil:
-    result[0].typ = disp.typ
+    result[0].typ() = disp.typ
     result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
     for i in 1..<result.len:
@@ -133,7 +133,7 @@ proc createDispatcher(s: PSym; g: ModuleGraph; idgen: IdGenerator): PSym =
   if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
   disp.ast[bodyPos] = newNodeI(nkEmpty, s.info)
-  disp.loc.r = ""
+  disp.loc.snippet = ""
   if s.typ.returnType != nil:
     if disp.ast.len > resultPos:
       disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, idgen)

compiler/closureiters.nim

@@ -18,7 +18,8 @@
 #    dec a
 #
 # Should be transformed to:
-#  STATE0:
+#  case :state
+#  of 0:
 #    if a > 0:
 #      echo "hi"
 #      :state = 1 # Next state
@@ -26,19 +27,14 @@
 #    else:
 #      :state = 2 # Next state
 #      break :stateLoop # Proceed to the next state
-#  STATE1:
+#  of 1:
 #    dec a
 #    :state = 0 # Next state
 #    break :stateLoop # Proceed to the next state
-#  STATE2:
+#  of 2:
 #    :state = -1 # End of execution
-
-# The transformation should play well with lambdalifting, however depending
-# on situation, it can be called either before or after lambdalifting
-# transformation. As such we behave slightly differently, when accessing
-# iterator state, or using temp variables. If lambdalifting did not happen,
-# we just create local variables, so that they will be lifted further on.
-# Otherwise, we utilize existing env, created by lambdalifting.
+#  else:
+#    return
 
 # Lambdalifting treats :state variable specially, it should always end up
 # as the first field in env. Currently C codegen depends on this behavior.
@@ -104,12 +100,13 @@
 # Is transformed to (yields are left in place for example simplicity,
 #    in reality the code is subdivided even more, as described above):
 #
-# STATE0: # Try
+# case :state
+# of 0: # Try
 #   yield 0
 #   raise ...
 #   :state = 2 # What would happen should we not raise
 #   break :stateLoop
-# STATE1: # Except
+# of 1: # Except
 #   yield 1
 #   :tmpResult = 3           # Return
 #   :unrollFinally = true # Return
@@ -117,7 +114,7 @@
 #   break :stateLoop
 #   :state = 2 # What would happen should we not return
 #   break :stateLoop
-# STATE2: # Finally
+# of 2: # Finally
 #   yield 2
 #   if :unrollFinally: # This node is created by `newEndFinallyNode`
 #     if :curExc.isNil:
@@ -130,6 +127,8 @@
 #       raise
 #   state = -1 # Goto next state. In this case we just exit
 #   break :stateLoop
+# else:
+#   return
 
 import
   ast, msgs, idents,
@@ -145,12 +144,11 @@ type
   Ctx = object
     g: ModuleGraph
     fn: PSym
-    stateVarSym: PSym # :state variable. nil if env already introduced by lambdalifting
     tmpResultSym: PSym # Used when we return, but finally has to interfere
     unrollFinallySym: PSym # Indicates that we're unrolling finally states (either exception happened or premature return)
     curExcSym: PSym # Current exception
 
-    states: seq[PNode] # The resulting states. Every state is an nkState node.
+    states: seq[tuple[label: int, body: PNode]] # The resulting states.
     blockLevel: int # Temp used to transform break and continue stmts
     stateLoopLabel: PSym # Label to break on, when jumping between states.
     exitStateIdx: int # index of the last state
@@ -162,17 +160,18 @@ type
     nearestFinally: int # Index of the nearest finally block. For try/except it
                     # is their finally. For finally it is parent finally. Otherwise -1
     idgen: IdGenerator
+    varStates: Table[ItemId, int] # Used to detect if local variable belongs to multiple states
 
 const
   nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
             nkCommentStmt, nkMixinStmt, nkBindStmt} + procDefs
+  emptyStateLabel = -1
+  localNotSeen = -1
+  localRequiresLifting = -2
 
 proc newStateAccess(ctx: var Ctx): PNode =
-  if ctx.stateVarSym.isNil:
-    result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)),
-        getStateField(ctx.g, ctx.fn), ctx.fn.info)
-  else:
-    result = newSymNode(ctx.stateVarSym)
+  result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)),
+      getStateField(ctx.g, ctx.fn), ctx.fn.info)
 
 proc newStateAssgn(ctx: var Ctx, toValue: PNode): PNode =
   # Creates state assignment:
@@ -187,24 +186,18 @@ proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
   result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.idgen, ctx.fn, ctx.fn.info)
   result.typ = typ
-  assert(not typ.isNil)
+  result.flags.incl sfNoInit
+  assert(not typ.isNil, "Env var needs a type")
 
-  if not ctx.stateVarSym.isNil:
-    # We haven't gone through labmda lifting yet, so just create a local var,
-    # it will be lifted later
-    if ctx.tempVars.isNil:
-      ctx.tempVars = newNodeI(nkVarSection, ctx.fn.info)
-      addVar(ctx.tempVars, newSymNode(result))
-  else:
-    let envParam = getEnvParam(ctx.fn)
-    # let obj = envParam.typ.lastSon
-    result = addUniqueField(envParam.typ.elementType, result, ctx.g.cache, ctx.idgen)
+  let envParam = getEnvParam(ctx.fn)
+  # let obj = envParam.typ.lastSon
+  result = addUniqueField(envParam.typ.elementType, result, ctx.g.cache, ctx.idgen)
 
 proc newEnvVarAccess(ctx: Ctx, s: PSym): PNode =
-  if ctx.stateVarSym.isNil:
-    result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)), s, ctx.fn.info)
-  else:
-    result = newSymNode(s)
+  result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)), s, ctx.fn.info)
+
+proc newTempVarAccess(ctx: Ctx, s: PSym): PNode =
+  result = newSymNode(s, ctx.fn.info)
 
 proc newTmpResultAccess(ctx: var Ctx): PNode =
   if ctx.tmpResultSym.isNil:
@@ -228,10 +221,7 @@ proc newState(ctx: var Ctx, n, gotoOut: PNode): int =
 
   result = ctx.states.len
   let resLit = ctx.g.newIntLit(n.info, result)
-  let s = newNodeI(nkState, n.info)
-  s.add(resLit)
-  s.add(n)
-  ctx.states.add(s)
+  ctx.states.add((result, n))
   ctx.exceptionTable.add(ctx.curExcHandlingState)
 
   if not gotoOut.isNil:
@@ -250,9 +240,18 @@ proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   if result.len == 0 or result[^1].kind != nkGotoState:
     result.add(gotoOut)
 
-proc newTempVar(ctx: var Ctx, typ: PType): PSym =
-  result = ctx.newEnvVar(":tmpSlLower" & $ctx.tempVarId, typ)
+proc newTempVarDef(ctx: Ctx, s: PSym, initialValue: PNode): PNode =
+  var v = initialValue
+  if v == nil:
+    v = ctx.g.emptyNode
+  newTree(nkVarSection, newTree(nkIdentDefs, newSymNode(s), ctx.g.emptyNode, v))
+
+proc newTempVar(ctx: var Ctx, typ: PType, parent: PNode, initialValue: PNode = nil): PSym =
+  result = newSym(skVar, getIdent(ctx.g.cache, ":tmpSlLower" & $ctx.tempVarId), ctx.idgen, ctx.fn, ctx.fn.info)
   inc ctx.tempVarId
+  result.typ = typ
+  assert(not typ.isNil, "Temp var needs a type")
+  parent.add(ctx.newTempVarDef(result, initialValue))
 
 proc hasYields(n: PNode): bool =
   # TODO: This is very inefficient. It traverses the node, looking for nkYieldStmt.
@@ -263,8 +262,8 @@ proc hasYields(n: PNode): bool =
     result = false
   else:
     result = false
-    for c in n:
-      if c.hasYields:
+    for i in ord(n.kind == nkCast)..<n.len:
+      if n[i].hasYields:
         result = true
         break
 
@@ -312,12 +311,12 @@ proc newNullifyCurExc(ctx: var Ctx, info: TLineInfo): PNode =
   let curExc = ctx.newCurExcAccess()
   curExc.info = info
   let nilnode = newNode(nkNilLit)
-  nilnode.typ = curExc.typ
+  nilnode.typ() = curExc.typ
   result = newTree(nkAsgn, curExc, nilnode)
 
 proc newOr(g: ModuleGraph, a, b: PNode): PNode {.inline.} =
   result = newTree(nkCall, newSymNode(g.getSysMagic(a.info, "or", mOr)), a, b)
-  result.typ = g.getSysType(a.info, tyBool)
+  result.typ() = g.getSysType(a.info, tyBool)
   result.info = a.info
 
 proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
@@ -335,7 +334,7 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
             newSymNode(g.getSysMagic(c.info, "of", mOf)),
             g.callCodegenProc("getCurrentException"),
             c[i])
-          nextCond.typ = ctx.g.getSysType(c.info, tyBool)
+          nextCond.typ() = ctx.g.getSysType(c.info, tyBool)
           nextCond.info = c.info
 
           if cond.isNil:
@@ -424,21 +423,20 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
 
   result.res = n
 
-
-proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
+proc newTempVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
   if isEmptyType(v.typ):
     result = v
   else:
-    result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
+    result = newTree(nkFastAsgn, ctx.newTempVarAccess(s), v)
     result.info = v.info
 
 proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
   if input.kind == nkStmtListExpr:
     let (st, res) = exprToStmtList(input)
     output.add(st)
-    output.add(ctx.newEnvVarAsgn(sym, res))
+    output.add(ctx.newTempVarAsgn(sym, res))
   else:
-    output.add(ctx.newEnvVarAsgn(sym, input))
+    output.add(ctx.newTempVarAsgn(sym, input))
 
 proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode =
   result = newNodeI(nkStmtList, exprBody.info)
@@ -446,7 +444,17 @@ proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode =
 
 proc newNotCall(g: ModuleGraph; e: PNode): PNode =
   result = newTree(nkCall, newSymNode(g.getSysMagic(e.info, "not", mNot), e.info), e)
-  result.typ = g.getSysType(e.info, tyBool)
+  result.typ() = g.getSysType(e.info, tyBool)
+
+proc boolLit(g: ModuleGraph; info: TLineInfo; value: bool): PNode =
+  result = newIntLit(g, info, ord value)
+  result.typ() = getSysType(g, info, tyBool)
+
+proc captureVar(c: var Ctx, s: PSym) =
+  if c.varStates.getOrDefault(s.itemId) != localRequiresLifting:
+    c.varStates[s.itemId] = localRequiresLifting # Mark this variable for lifting
+    let e = getEnvParam(c.fn)
+    discard addField(e.typ.elementType, s, c.g.cache, c.idgen)
 
 proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
   result = n
@@ -478,7 +486,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
       result = newNodeI(nkStmtListExpr, n.info)
       if n.typ.isNil: internalError(ctx.g.config, "lowerStmtListExprs: constr typ.isNil")
-      result.typ = n.typ
+      result.typ() = n.typ
 
       for i in 0..<n.len:
         case n[i].kind
@@ -504,9 +512,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       var tmp: PSym = nil
       let isExpr = not isEmptyType(n.typ)
       if isExpr:
-        tmp = ctx.newTempVar(n.typ)
         result = newNodeI(nkStmtListExpr, n.info)
-        result.typ = n.typ
+        result.typ() = n.typ
+        tmp = ctx.newTempVar(n.typ, result)
       else:
         result = newNodeI(nkStmtList, n.info)
 
@@ -557,7 +565,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         else:
           internalError(ctx.g.config, "lowerStmtListExpr(nkIf): " & $branch.kind)
 
-      if isExpr: result.add(ctx.newEnvVarAccess(tmp))
+      if isExpr: result.add(ctx.newTempVarAccess(tmp))
 
   of nkTryStmt, nkHiddenTryStmt:
     var ns = false
@@ -570,8 +578,8 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
       if isExpr:
         result = newNodeI(nkStmtListExpr, n.info)
-        result.typ = n.typ
-        let tmp = ctx.newTempVar(n.typ)
+        result.typ() = n.typ
+        let tmp = ctx.newTempVar(n.typ, result)
 
         n[0] = ctx.convertExprBodyToAsgn(n[0], tmp)
         for i in 1..<n.len:
@@ -587,7 +595,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           else:
             internalError(ctx.g.config, "lowerStmtListExpr(nkTryStmt): " & $branch.kind)
         result.add(n)
-        result.add(ctx.newEnvVarAccess(tmp))
+        result.add(ctx.newTempVarAccess(tmp))
 
   of nkCaseStmt:
     var ns = false
@@ -600,9 +608,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       let isExpr = not isEmptyType(n.typ)
 
       if isExpr:
-        let tmp = ctx.newTempVar(n.typ)
         result = newNodeI(nkStmtListExpr, n.info)
-        result.typ = n.typ
+        result.typ() = n.typ
+        let tmp = ctx.newTempVar(n.typ, result)
 
         if n[0].kind == nkStmtListExpr:
           let (st, ex) = exprToStmtList(n[0])
@@ -619,7 +627,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           else:
             internalError(ctx.g.config, "lowerStmtListExpr(nkCaseStmt): " & $branch.kind)
         result.add(n)
-        result.add(ctx.newEnvVarAccess(tmp))
+        result.add(ctx.newTempVarAccess(tmp))
       elif n[0].kind == nkStmtListExpr:
         result = newNodeI(nkStmtList, n.info)
         let (st, ex) = exprToStmtList(n[0])
@@ -638,7 +646,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
       if isExpr:
         result = newNodeI(nkStmtListExpr, n.info)
-        result.typ = n.typ
+        result.typ() = n.typ
       else:
         result = newNodeI(nkStmtList, n.info)
 
@@ -649,10 +657,10 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           result.add(st)
           cond = ex
 
-        let tmp = ctx.newTempVar(cond.typ)
-        result.add(ctx.newEnvVarAsgn(tmp, cond))
+        let tmp = ctx.newTempVar(cond.typ, result, cond)
+        # result.add(ctx.newTempVarAsgn(tmp, cond))
 
-        var check = ctx.newEnvVarAccess(tmp)
+        var check = ctx.newTempVarAccess(tmp)
         if n[0].sym.magic == mOr:
           check = ctx.g.newNotCall(check)
 
@@ -662,12 +670,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           let (st, ex) = exprToStmtList(cond)
           ifBody.add(st)
           cond = ex
-        ifBody.add(ctx.newEnvVarAsgn(tmp, cond))
+        ifBody.add(ctx.newTempVarAsgn(tmp, cond))
 
         let ifBranch = newTree(nkElifBranch, check, ifBody)
         let ifNode = newTree(nkIfStmt, ifBranch)
         result.add(ifNode)
-        result.add(ctx.newEnvVarAccess(tmp))
+        result.add(ctx.newTempVarAccess(tmp))
       else:
         for i in 0..<n.len:
           if n[i].kind == nkStmtListExpr:
@@ -676,9 +684,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
             n[i] = ex
 
           if n[i].kind in nkCallKinds: # XXX: This should better be some sort of side effect tracking
-            let tmp = ctx.newTempVar(n[i].typ)
-            result.add(ctx.newEnvVarAsgn(tmp, n[i]))
-            n[i] = ctx.newEnvVarAccess(tmp)
+            let tmp = ctx.newTempVar(n[i].typ, result, n[i])
+            # result.add(ctx.newTempVarAsgn(tmp, n[i]))
+            n[i] = ctx.newTempVarAccess(tmp)
 
         result.add(n)
 
@@ -694,6 +702,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         let (st, ex) = exprToStmtList(c[^1])
         result.add(st)
         c[^1] = ex
+      for i in 0 .. c.len - 3:
+        if c[i].kind == nkSym:
+          let s = c[i].sym
+          if sfForceLift in s.flags:
+            ctx.captureVar(s)
+
       result.add(varSect)
 
   of nkDiscardStmt, nkReturnStmt, nkRaiseStmt:
@@ -718,7 +732,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
     if ns:
       needsSplit = true
       result = newNodeI(nkStmtListExpr, n.info)
-      result.typ = n.typ
+      result.typ() = n.typ
       let (st, ex) = exprToStmtList(n[^1])
       result.add(st)
       n[^1] = ex
@@ -779,7 +793,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         let check = newTree(nkIfStmt, branch)
         let newBody = newTree(nkStmtList, st, check, n[1])
 
-        n[0] = newSymNode(ctx.g.getSysSym(n[0].info, "true"))
+        n[0] = ctx.g.boolLit(n[0].info, true)
         n[1] = newBody
 
   of nkDotExpr, nkCheckedFieldExpr:
@@ -788,7 +802,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
     if ns:
       needsSplit = true
       result = newNodeI(nkStmtListExpr, n.info)
-      result.typ = n.typ
+      result.typ() = n.typ
       let (st, ex) = exprToStmtList(n[0])
       result.add(st)
       n[0] = ex
@@ -800,10 +814,10 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
     if ns:
       needsSplit = true
       result = newNodeI(nkStmtListExpr, n.info)
-      result.typ = n.typ
+      result.typ() = n.typ
       let (st, ex) = exprToStmtList(n[1])
       n.transitionSonsKind(nkBlockStmt)
-      n.typ = nil
+      n.typ() = nil
       n[1] = st
       result.add(n)
       result.add(ex)
@@ -824,9 +838,9 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   #         raise
   let curExc = ctx.newCurExcAccess()
   let nilnode = newNode(nkNilLit)
-  nilnode.typ = curExc.typ
+  nilnode.typ() = curExc.typ
   let cmp = newTree(nkCall, newSymNode(ctx.g.getSysMagic(info, "==", mEqRef), info), curExc, nilnode)
-  cmp.typ = ctx.g.getSysType(info, tyBool)
+  cmp.typ() = ctx.g.getSysType(info, tyBool)
 
   let retStmt =
     if ctx.nearestFinally == 0:
@@ -1133,10 +1147,10 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
     let label = stateIdx
     if label == ctx.exitStateIdx: break
     var newLabel = label
-    if label == -1:
+    if label == emptyStateLabel:
       newLabel = ctx.exitStateIdx
     else:
-      let fs = skipStmtList(ctx, ctx.states[label][1])
+      let fs = skipStmtList(ctx, ctx.states[label].body)
       if fs.kind == nkGotoState:
         newLabel = fs[0].intVal.int
     if label == newLabel: break
@@ -1145,7 +1159,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
     if maxJumps == 0:
       assert(false, "Internal error")
 
-  result = ctx.states[stateIdx][0].intVal.int
+  result = ctx.states[stateIdx].label
 
 proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
@@ -1171,11 +1185,11 @@ proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: P
 
 proc createExceptionTable(ctx: var Ctx): PNode {.inline.} =
   result = newNodeI(nkBracket, ctx.fn.info)
-  result.typ = ctx.g.newArrayType(ctx.exceptionTable.len, ctx.g.getSysType(ctx.fn.info, tyInt16), ctx.idgen, ctx.fn)
+  result.typ() = ctx.g.newArrayType(ctx.exceptionTable.len, ctx.g.getSysType(ctx.fn.info, tyInt16), ctx.idgen, ctx.fn)
 
   for i in ctx.exceptionTable:
     let elem = newIntNode(nkIntLit, i)
-    elem.typ = ctx.g.getSysType(ctx.fn.info, tyInt16)
+    elem.typ() = ctx.g.getSysType(ctx.fn.info, tyInt16)
     result.add(elem)
 
 proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
@@ -1198,7 +1212,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
     let getNextState = newTree(nkBracketExpr,
       ctx.createExceptionTable(),
       ctx.newStateAccess())
-    getNextState.typ = intTyp
+    getNextState.typ() = intTyp
 
     # :state = exceptionTable[:state]
     result.add(ctx.newStateAssgn(getNextState))
@@ -1209,7 +1223,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
       ctx.g.getSysMagic(info, "==", mEqI).newSymNode(),
       ctx.newStateAccess(),
       newIntTypeNode(0, intTyp))
-    cond.typ = boolTyp
+    cond.typ() = boolTyp
 
     let raiseStmt = newTree(nkRaiseStmt, ctx.g.emptyNode)
     let ifBranch = newTree(nkElifBranch, cond, raiseStmt)
@@ -1222,7 +1236,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
       ctx.g.getSysMagic(info, "<", mLtI).newSymNode,
       newIntTypeNode(0, intTyp),
       ctx.newStateAccess())
-    cond.typ = boolTyp
+    cond.typ() = boolTyp
 
     let asgn = newTree(nkAsgn, ctx.newUnrollFinallyAccess(info), cond)
     result.add(asgn)
@@ -1233,12 +1247,12 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
       ctx.g.getSysMagic(info, "<", mLtI).newSymNode,
       ctx.newStateAccess(),
       newIntTypeNode(0, intTyp))
-    cond.typ = boolTyp
+    cond.typ() = boolTyp
 
     let negateState = newTree(nkCall,
       ctx.g.getSysMagic(info, "-", mUnaryMinusI).newSymNode,
       ctx.newStateAccess())
-    negateState.typ = intTyp
+    negateState.typ() = intTyp
 
     let ifBranch = newTree(nkElifBranch, cond, ctx.newStateAssgn(negateState))
     let ifStmt = newTree(nkIfStmt, ifBranch)
@@ -1263,30 +1277,18 @@ proc wrapIntoTryExcept(ctx: var Ctx, n: PNode): PNode {.inline.} =
 proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   # while true:
   #   block :stateLoop:
-  #     gotoState :state
   #     local vars decl (if needed)
   #     body # Might get wrapped in try-except
   let loopBody = newNodeI(nkStmtList, n.info)
-  result = newTree(nkWhileStmt, newSymNode(ctx.g.getSysSym(n.info, "true")), loopBody)
+  result = newTree(nkWhileStmt, ctx.g.boolLit(n.info, true), loopBody)
   result.info = n.info
 
   let localVars = newNodeI(nkStmtList, n.info)
-  if not ctx.stateVarSym.isNil:
-    let varSect = newNodeI(nkVarSection, n.info)
-    addVar(varSect, newSymNode(ctx.stateVarSym))
-    localVars.add(varSect)
-
-    if not ctx.tempVars.isNil:
-      localVars.add(ctx.tempVars)
 
   let blockStmt = newNodeI(nkBlockStmt, n.info)
   blockStmt.add(newSymNode(ctx.stateLoopLabel))
 
-  let gs = newNodeI(nkGotoState, n.info)
-  gs.add(ctx.newStateAccess())
-  gs.add(ctx.g.newIntLit(n.info, ctx.states.len - 1))
-
-  var blockBody = newTree(nkStmtList, gs, localVars, n)
+  var blockBody = newTree(nkStmtList, localVars, n)
   if ctx.hasExceptions:
     blockBody = ctx.wrapIntoTryExcept(blockBody)
 
@@ -1299,29 +1301,28 @@ proc deleteEmptyStates(ctx: var Ctx) =
 
   # Apply new state indexes and mark unused states with -1
   var iValid = 0
-  for i, s in ctx.states:
-    let body = skipStmtList(ctx, s[1])
+  for i, s in ctx.states.mpairs:
+    let body = skipStmtList(ctx, s.body)
     if body.kind == nkGotoState and i != ctx.states.len - 1 and i != 0:
       # This is an empty state. Mark with -1.
-      s[0].intVal = -1
+      s.label = emptyStateLabel
     else:
-      s[0].intVal = iValid
+      s.label = iValid
       inc iValid
 
   for i, s in ctx.states:
-    let body = skipStmtList(ctx, s[1])
+    let body = skipStmtList(ctx, s.body)
     if body.kind != nkGotoState or i == 0:
-      discard ctx.skipThroughEmptyStates(s)
+      discard ctx.skipThroughEmptyStates(s.body)
       let excHandlState = ctx.exceptionTable[i]
       if excHandlState < 0:
         ctx.exceptionTable[i] = -ctx.skipEmptyStates(-excHandlState)
       elif excHandlState != 0:
         ctx.exceptionTable[i] = ctx.skipEmptyStates(excHandlState)
 
-  var i = 0
+  var i = 1 # ignore the entry and the exit
   while i < ctx.states.len - 1:
-    let fs = skipStmtList(ctx, ctx.states[i][1])
-    if fs.kind == nkGotoState and i != 0:
+    if ctx.states[i].label == emptyStateLabel:
       ctx.states.delete(i)
       ctx.exceptionTable.delete(i)
     else:
@@ -1430,15 +1431,67 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
 
+proc detectCapturedVars(c: var Ctx, n: PNode, stateIdx: int) =
+  case n.kind
+  of nkSym:
+    let s = n.sym
+    if s.kind in {skResult, skVar, skLet, skForVar, skTemp} and sfGlobal notin s.flags and s.owner == c.fn:
+      let vs = c.varStates.getOrDefault(s.itemId, localNotSeen)
+      if vs == localNotSeen: # First seing this variable
+        c.varStates[s.itemId] = stateIdx
+      elif vs == localRequiresLifting:
+        discard # Sym already marked
+      elif vs != stateIdx:
+        c.captureVar(s)
+  of nkReturnStmt:
+    if n[0].kind in {nkAsgn, nkFastAsgn, nkSinkAsgn}:
+      # we have a `result = result` expression produced by the closure
+      # transform, let's not touch the LHS in order to make the lifting pass
+      # correct when `result` is lifted
+      detectCapturedVars(c, n[0][1], stateIdx)
+    else:
+      detectCapturedVars(c, n[0], stateIdx)
+  else:
+    for i in 0 ..< n.safeLen:
+      detectCapturedVars(c, n[i], stateIdx)
+
+proc detectCapturedVars(c: var Ctx) =
+  for i, s in c.states:
+    detectCapturedVars(c, s.body, i)
+
+proc liftLocals(c: var Ctx, n: PNode): PNode =
+  result = n
+  case n.kind
+  of nkSym:
+    let s = n.sym
+    if c.varStates.getOrDefault(s.itemId) == localRequiresLifting:
+      # lift
+      let e = getEnvParam(c.fn)
+      let field = getFieldFromObj(e.typ.elementType, s)
+      assert(field != nil)
+      result = rawIndirectAccess(newSymNode(e), field, n.info)
+    # elif c.varStates.getOrDefault(s.itemId, localNotSeen) != localNotSeen:
+    #   echo "Not lifting ", s.name.s
+
+  of nkReturnStmt:
+    if n[0].kind in {nkAsgn, nkFastAsgn, nkSinkAsgn}:
+      # we have a `result = result` expression produced by the closure
+      # transform, let's not touch the LHS in order to make the lifting pass
+      # correct when `result` is lifted
+      n[0][1] = liftLocals(c, n[0][1])
+    else:
+      n[0] = liftLocals(c, n[0])
+  else:
+    for i in 0 ..< n.safeLen:
+      n[i] = liftLocals(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n: PNode): PNode =
   var ctx = Ctx(g: g, fn: fn, idgen: idgen)
 
-  if getEnvParam(fn).isNil:
-    # Lambda lifting was not done yet. Use temporary :state sym, which will
-    # be handled specially by lambda lifting. Local temp vars (if needed)
-    # should follow the same logic.
-    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), idgen, fn, fn.info)
-    ctx.stateVarSym.typ = g.createClosureIterStateType(fn, idgen)
+  # The transformation should always happen after at least partial lambdalifting
+  # is performed, so that the closure iter environment is always created upfront.
+  doAssert(getEnvParam(fn) != nil, "Env param not created before iter transformation")
+
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), idgen, fn, fn.info)
   var pc = PreprocessContext(finallys: @[], config: g.config, idgen: idgen)
   var n = preprocess(pc, n.toStmtList)
@@ -1460,17 +1513,21 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
   # Optimize empty states away
   ctx.deleteEmptyStates()
 
-  # Make new body by concatenating the list of states
-  result = newNodeI(nkStmtList, n.info)
-  for s in ctx.states:
-    assert(s.len == 2)
-    let body = s[1]
-    s.sons.del(1)
-    result.add(s)
-    result.add(body)
+  let caseDispatcher = newTreeI(nkCaseStmt, n.info,
+      ctx.newStateAccess())
 
-  result = ctx.transformStateAssignments(result)
-  result = ctx.wrapIntoStateLoop(result)
+  # Lamdalifting will not touch our locals, it is our responsibility to lift those that
+  # need it.
+  detectCapturedVars(ctx)
+
+  for s in ctx.states:
+    let body = ctx.transformStateAssignments(s.body)
+    caseDispatcher.add newTreeI(nkOfBranch, body.info, g.newIntLit(body.info, s.label), body)
+
+  caseDispatcher.add newTreeI(nkElse, n.info, newTreeI(nkReturnStmt, n.info, g.emptyNode))
+
+  result = wrapIntoStateLoop(ctx, caseDispatcher)
+  result = liftLocals(ctx, result)
 
   when false:
     echo "TRANSFORM TO STATES: "
@@ -1479,3 +1536,5 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
     echo "exception table:"
     for i, e in ctx.exceptionTable:
       echo i, " -> ", e
+
+    echo "ENV: ", renderTree(getEnvParam(fn).typ.elementType.n)

compiler/commands.nim

@@ -24,7 +24,7 @@ bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
 bootSwitch(usedGoGC, defined(gogc), "--gc:go")
 bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
-import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs]
+import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs, enumutils]
 import
   msgs, options, nversion, condsyms, extccomp, platform,
   wordrecg, nimblecmd, lineinfos, pathutils
@@ -248,6 +248,7 @@ const
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console', 'lib' or 'staticlib' expected, but '$1' found"
   errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found"
+  errInvalidFeatureButXFound = Feature.toSeq.map(proc(val:Feature): string = "'$1'" % $val).join(", ") & " expected, but '$1' found"
 
 template warningOptionNoop(switch: string) =
   warningDeprecated(conf, info, "'$#' is deprecated, now a noop" % switch)
@@ -303,6 +304,12 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
     else:
       result = false
       localError(conf, info, errInvalidExceptionSystem % arg)
+  of "experimental":
+    try:
+      result = conf.features.contains parseEnum[Feature](arg)
+    except ValueError:
+      result = false
+      localError(conf, info, errInvalidFeatureButXFound % arg)
   else:
     result = false
     invalidCmdLineOption(conf, passCmd1, switch, info)
@@ -462,7 +469,6 @@ proc handleCmdInput*(conf: ConfigRef) =
 proc parseCommand*(command: string): Command =
   case command.normalize
   of "c", "cc", "compile", "compiletoc": cmdCompileToC
-  of "nir": cmdCompileToNir
   of "cpp", "compiletocpp": cmdCompileToCpp
   of "objc", "compiletooc": cmdCompileToOC
   of "js", "compiletojs": cmdCompileToJS
@@ -500,7 +506,6 @@ proc setCmd*(conf: ConfigRef, cmd: Command) =
   of cmdCompileToCpp: conf.backend = backendCpp
   of cmdCompileToOC: conf.backend = backendObjc
   of cmdCompileToJS: conf.backend = backendJs
-  of cmdCompileToNir: conf.backend = backendNir
   else: discard
 
 proc setCommandEarly*(conf: ConfigRef, command: string) =

compiler/concepts.nim

@@ -11,7 +11,7 @@
 ## for details. Note this is a first implementation and only the "Concept matching"
 ## section has been implemented.
 
-import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types
+import ast, semdata, lookups, lineinfos, idents, msgs, renderer, types, layeredtable
 
 import std/intsets
 
@@ -309,7 +309,7 @@ proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool =
     # error was reported earlier.
     result = false
 
-proc conceptMatch*(c: PContext; concpt, arg: PType; bindings: var TypeMapping; invocation: PType): bool =
+proc conceptMatch*(c: PContext; concpt, arg: PType; bindings: var LayeredIdTable; invocation: PType): bool =
   ## Entry point from sigmatch. 'concpt' is the concept we try to match (here still a PType but
   ## we extract its AST via 'concpt.n.lastSon'). 'arg' is the type that might fulfill the
   ## concept's requirements. If so, we return true and fill the 'bindings' with pairs of
@@ -328,16 +328,16 @@ proc conceptMatch*(c: PContext; concpt, arg: PType; bindings: var TypeMapping; i
           dest = existingBinding(m, dest)
           if dest == nil or dest.kind != tyGenericParam: break
         if dest != nil:
-          bindings.idTablePut(a, dest)
+          bindings.put(a, dest)
           when logBindings: echo "A bind ", a, " ", dest
       else:
-        bindings.idTablePut(a, b)
+        bindings.put(a, b)
         when logBindings: echo "B bind ", a, " ", b
     # we have a match, so bind 'arg' itself to 'concpt':
-    bindings.idTablePut(concpt, arg)
+    bindings.put(concpt, arg)
     # invocation != nil means we have a non-atomic concept:
     if invocation != nil and arg.kind == tyGenericInst and invocation.kidsLen == arg.kidsLen-1:
       # bind even more generic parameters
       assert invocation.kind == tyGenericInvocation
       for i in FirstGenericParamAt ..< invocation.kidsLen:
-        bindings.idTablePut(invocation[i], arg[i])
+        bindings.put(invocation[i], arg[i])

compiler/condsyms.nim

@@ -166,3 +166,8 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasWarnStdPrefix")
 
   defineSymbol("nimHasVtables")
+  defineSymbol("nimHasGenericsOpenSym2")
+  defineSymbol("nimHasGenericsOpenSym3")
+  defineSymbol("nimHasJsNoLambdaLifting")
+  defineSymbol("nimHasDefaultFloatRoundtrip")
+  defineSymbol("nimHasXorSet")

compiler/dfa.nim

@@ -46,10 +46,10 @@ type
     case isTryBlock: bool
     of false:
       label: PSym
-      breakFixups: seq[(TPosition, seq[PNode])] #Contains the gotos for the breaks along with their pending finales
+      breakFixups: seq[(TPosition, seq[PNode])] # Contains the gotos for the breaks along with their pending finales
     of true:
       finale: PNode
-      raiseFixups: seq[TPosition] #Contains the gotos for the raises
+      raiseFixups: seq[TPosition] # Contains the gotos for the raises
 
   Con = object
     code: ControlFlowGraph
@@ -181,14 +181,6 @@ proc genIf(c: var Con, n: PNode) =
     goto Lend3
   L3:
     D
-    goto Lend3 # not eliminated to simplify the join generation
-  Lend3:
-    join F3
-  Lend2:
-    join F2
-  Lend:
-    join F1
-
   ]#
   var endings: seq[TPosition] = @[]
   let oldInteresting = c.interestingInstructions
@@ -213,7 +205,6 @@ proc genAndOr(c: var Con; n: PNode) =
   #   fork lab1
   #   asgn dest, b
   # lab1:
-  #   join F1
   c.gen(n[1])
   forkT:
     c.gen(n[2])
@@ -324,7 +315,7 @@ proc genRaise(c: var Con; n: PNode) =
       if c.blocks[i].isTryBlock:
         genBreakOrRaiseAux(c, i, n)
         return
-    assert false #Unreachable
+    assert false # Unreachable
   else:
     genNoReturn(c)
 
@@ -390,7 +381,6 @@ proc genCall(c: var Con; n: PNode) =
     # fork lab1
     # goto exceptionHandler (except or finally)
     # lab1:
-    # join F1
     forkT:
       for i in countdown(c.blocks.high, 0):
         if c.blocks[i].isTryBlock:

compiler/docgen.nim

@@ -115,7 +115,7 @@ proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, s
 
 proc addRstFileIndex(d: PDoc, fileIndex: lineinfos.FileIndex): rstast.FileIndex =
   let invalid = rstast.FileIndex(-1)
-  result = d.nimToRstFid.getOrDefault(fileIndex, default = invalid)
+  result = d.nimToRstFid.getOrDefault(fileIndex, invalid)
   if result == invalid:
     let fname = toFullPath(d.conf, fileIndex)
     result = addFilename(d.sharedState, fname)
@@ -830,8 +830,8 @@ proc getName(n: PNode): string =
   of nkAccQuoted:
     result = "`"
     for i in 0..<n.len: result.add(getName(n[i]))
-    result = "`"
-  of nkOpenSymChoice, nkClosedSymChoice:
+    result.add('`')
+  of nkOpenSymChoice, nkClosedSymChoice, nkOpenSym:
     result = getName(n[0])
   else:
     result = ""
@@ -849,7 +849,7 @@ proc getNameIdent(cache: IdentCache; n: PNode): PIdent =
     var r = ""
     for i in 0..<n.len: r.add(getNameIdent(cache, n[i]).s)
     result = getIdent(cache, r)
-  of nkOpenSymChoice, nkClosedSymChoice:
+  of nkOpenSymChoice, nkClosedSymChoice, nkOpenSym:
     result = getNameIdent(cache, n[0])
   else:
     result = nil
@@ -863,7 +863,7 @@ proc getRstName(n: PNode): PRstNode =
   of nkAccQuoted:
     result = getRstName(n[0])
     for i in 1..<n.len: result.text.add(getRstName(n[i]).text)
-  of nkOpenSymChoice, nkClosedSymChoice:
+  of nkOpenSymChoice, nkClosedSymChoice, nkOpenSym:
     result = getRstName(n[0])
   else:
     result = nil
@@ -1206,7 +1206,7 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind, nonExports = false):
         var param = %{"name": %($genericParam)}
         if genericParam.sym.typ.len > 0:
           param["types"] = newJArray()
-        param["types"].add %($genericParam.sym.typ.elementType)
+          param["types"] = %($genericParam.sym.typ.elementType)
         result.json["signature"]["genericParams"].add param
   if optGenIndex in d.conf.globalOptions:
     genItem(d, n, nameNode, k, kForceExport)
@@ -1320,7 +1320,7 @@ proc documentEffect(cache: IdentCache; n, x: PNode, effectType: TSpecialWord, id
       if t.startsWith("ref "): t = substr(t, 4)
       effects[i] = newIdentNode(getIdent(cache, t), n.info)
       # set the type so that the following analysis doesn't screw up:
-      effects[i].typ = real[i].typ
+      effects[i].typ() = real[i].typ
 
     result = newTreeI(nkExprColonExpr, n.info,
       newIdentNode(getIdent(cache, $effectType), n.info), effects)

compiler/evalffi.nim

@@ -275,7 +275,7 @@ proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   # the nkPar node:
   if n.isNil:
     result = newNode(nkTupleConstr)
-    result.typ = typ
+    result.typ() = typ
     if typ.n.isNil:
       internalError(conf, "cannot unpack unnamed tuple")
     unpackObjectAdd(conf, x, typ.n, result)
@@ -298,7 +298,7 @@ proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
 proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   if n.isNil:
     result = newNode(nkBracket)
-    result.typ = typ
+    result.typ() = typ
     newSeq(result.sons, lengthOrd(conf, typ).toInt)
   else:
     result = n
@@ -319,7 +319,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   template aw(k, v, field: untyped): untyped =
     if n.isNil:
       result = newNode(k)
-      result.typ = typ
+      result.typ() = typ
     else:
       # check we have the right field:
       result = n
@@ -333,12 +333,12 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   template setNil() =
     if n.isNil:
       result = newNode(nkNilLit)
-      result.typ = typ
+      result.typ() = typ
     else:
       reset n[]
       result = n
       result[] = TNode(kind: nkNilLit)
-      result.typ = typ
+      result.typ() = typ
 
   template awi(kind, v: untyped): untyped = aw(kind, v, intVal)
   template awf(kind, v: untyped): untyped = aw(kind, v, floatVal)
@@ -427,7 +427,7 @@ proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
     # cast through a pointer needs a new inner object:
     let y = if x.kind == nkRefTy: newNodeI(nkRefTy, x.info, 1)
             else: x.copyTree
-    y.typ = x.typ
+    y.typ() = x.typ
     result = unpack(conf, a, destTyp, y)
     dealloc a
 
@@ -481,7 +481,7 @@ proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
     if aTyp.isNil:
       internalAssert conf, i+1 < fntyp.len
       aTyp = fntyp[i+1]
-      args[i+start].typ = aTyp
+      args[i+start].typ() = aTyp
     sig[i] = mapType(conf, aTyp)
     if sig[i].isNil: globalError(conf, info, "cannot map FFI type")
 

compiler/evaltempl.nim

@@ -33,6 +33,19 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
     let x = param
     if x.kind == nkArgList:
       for y in items(x): result.add(y)
+    elif nfDefaultRefsParam in x.flags:
+      # value of default param needs to be evaluated like template body
+      # if it contains other template params
+      var res: PNode
+      if isAtom(x):
+        res = newNodeI(nkPar, x.info)
+        evalTemplateAux(x, actual, c, res)
+        if res.len == 1: res = res[0]
+      else:
+        res = copyNode(x)
+        for i in 0..<x.safeLen:
+          evalTemplateAux(x[i], actual, c, res)
+      result.add res
     else:
       result.add copyTree(x)
 
@@ -51,11 +64,12 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         if x == nil:
           x = copySym(s, c.idgen)
           # sem'check needs to set the owner properly later, see bug #9476
-          x.owner = nil # c.genSymOwner
+          setOwner(x, nil) # c.genSymOwner
           #if x.kind == skParam and x.owner.kind == skModule:
           #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
         if sfGenSym in s.flags:
+          # TODO: getIdent(c.ic, "`" & x.name.s & "`gensym" & $c.instID)
           result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $c.instID),
             if c.instLines: actual.info else: templ.info)
         else:
@@ -168,7 +182,7 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
     d.add newSymNode(sym, info)
     result.add d
     result.add res
-    result.typ = res.typ
+    result.typ() = res.typ
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
                    conf: ConfigRef;

compiler/expanddefaults.nim

@@ -125,7 +125,7 @@ proc expandDefault(t: PType; info: TLineInfo): PNode =
   of tyString:
     result = newZero(t, info, nkStrLit)
   of tyNone, tyEmpty, tyUntyped, tyTyped, tyTypeDesc,
-     tyNil, tyGenericInvocation, tyProxy, tyBuiltInTypeClass,
+     tyNil, tyGenericInvocation, tyError, tyBuiltInTypeClass,
      tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass,
      tyAnd, tyOr, tyNot, tyAnything, tyConcept, tyIterable, tyForward:
     result = newZero(t, info, nkEmpty) # bug indicator

compiler/extccomp.nim

@@ -63,7 +63,7 @@ type
 
 # Configuration settings for various compilers.
 # When adding new compilers, the cmake sources could be a good reference:
-# http://cmake.org/gitweb?p=cmake.git;a=tree;f=Modules/Platform;
+# https://cmake.org/gitweb?p=cmake.git;a=tree;f=Modules/Platform;
 
 template compiler(name, settings: untyped): untyped =
   proc name: TInfoCC {.compileTime.} = settings
@@ -162,7 +162,11 @@ compiler vcc:
     linkerExe: "cl",
     linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui /nologo $options",
     includeCmd: " /I",
-    linkDirCmd: " /LIBPATH:",
+    # HACK: we call `cl` so we have to pass `/link` for linker options,
+    # but users may still want to pass arguments to `cl` (see #14221)
+    # to deal with this, we add `/link` before each `/LIBPATH`,
+    # the linker ignores extra `/link`s since it's an unrecognized argument
+    linkDirCmd: " /link /LIBPATH:",
     linkLibCmd: " $1.lib",
     debug: " /RTC1 /Z7 ",
     pic: "",
@@ -172,6 +176,22 @@ compiler vcc:
     cppXsupport: "",
     props: {hasCpp, hasAssume, hasDeclspec})
 
+# Nvidia CUDA NVCC Compiler
+compiler nvcc:
+  result = gcc()
+  result.name = "nvcc"
+  result.compilerExe = "nvcc"
+  result.cppCompiler = "nvcc"
+  result.compileTmpl = "-c -x cu -Xcompiler=\"$options\" $include -o $objfile $file"
+  result.linkTmpl = "$buildgui $builddll -o $exefile $objfiles -Xcompiler=\"$options\""
+
+# AMD HIPCC Compiler (rocm/cuda)
+compiler hipcc:
+  result = clang()
+  result.name = "hipcc"
+  result.compilerExe = "hipcc"
+  result.cppCompiler = "hipcc"
+
 compiler clangcl:
   result = vcc()
   result.name = "clang_cl"
@@ -285,7 +305,9 @@ const
     envcc(),
     icl(),
     icc(),
-    clangcl()]
+    clangcl(),
+    hipcc(),
+    nvcc()]
 
   hExt* = ".h"
 
@@ -319,7 +341,7 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
   var fullSuffix = suffix
   case conf.backend
   of backendCpp, backendJs, backendObjc: fullSuffix = "." & $conf.backend & suffix
-  of backendC, backendNir: discard
+  of backendC: discard
   of backendInvalid:
     # during parsing of cfg files; we don't know the backend yet, no point in
     # guessing wrong thing
@@ -759,7 +781,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
     # https://blog.molecular-matters.com/2017/05/09/deleting-pdb-files-locked-by-visual-studio/
     # and a bit about the .pdb format in case that is ever needed:
     # https://github.com/crosire/blink
-    # http://www.debuginfo.com/articles/debuginfomatch.html#pdbfiles
+    # https://www.debuginfo.com/articles/debuginfomatch.html#pdbfiles
     if conf.hcrOn and isVSCompatible(conf):
       let t = now()
       let pdb = output.string & "." & format(t, "MMMM-yyyy-HH-mm-") & $t.nanosecond & ".pdb"
@@ -982,10 +1004,11 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile =
   # works out of the box with `hashMainCompilationParams`.
   result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json")
 
-const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes
+const cacheVersion = "D20240927T193831" # update when `BuildCache` spec changes
 type BuildCache = object
   cacheVersion: string
   outputFile: string
+  outputLastModificationTime: string
   compile: seq[(string, string)]
   link: seq[string]
   linkcmd: string
@@ -1029,6 +1052,8 @@ proc writeJsonBuildInstructions*(conf: ConfigRef; deps: StringTableRef) =
           bcache.depfiles.add (path, $secureHashFile(path))
 
     bcache.nimexe = hashNimExe()
+    if fileExists(bcache.outputFile):
+      bcache.outputLastModificationTime = $getLastModificationTime(bcache.outputFile)
   conf.jsonBuildFile = conf.jsonBuildInstructionsFile
   conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty)
 
@@ -1049,6 +1074,8 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute
     # xxx optimize by returning false if stdin input was the same
   for (file, hash) in bcache.depfiles:
     if $secureHashFile(file) != hash: return true
+  if bcache.outputLastModificationTime != $getLastModificationTime(bcache.outputFile):
+    return true
 
 proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
   var bcache: BuildCache = default(BuildCache)
@@ -1065,7 +1092,7 @@ proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
       "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " %
       [outputCurrent, output, jsonFile.string])
   var cmds: TStringSeq = default(TStringSeq)
-  var prettyCmds: TStringSeq= default(TStringSeq)
+  var prettyCmds: TStringSeq = default(TStringSeq)
   let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
   for (name, cmd) in bcache.compile:
     cmds.add cmd

compiler/guards.nim

@@ -1104,7 +1104,7 @@ proc settype(n: PNode): PType =
 
 proc buildOf(it, loc: PNode; o: Operators): PNode =
   var s = newNodeI(nkCurly, it.info, it.len-1)
-  s.typ = settype(loc)
+  s.typ() = settype(loc)
   for i in 0..<it.len-1: s[i] = it[i]
   result = newNodeI(nkCall, it.info, 3)
   result[0] = newSymNode(o.opContains)
@@ -1165,8 +1165,12 @@ proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
   if check[1].kind == nkCurly:
     result = copyTree(check)
     if access.kind == nkDotExpr:
+      # change the access to the discriminator field access
       var a = copyTree(access)
+      # set field name to discriminator field name
       a[1] = check[2]
+      # set discriminator field type: important for `neg`
+      a.typ() = check[2].typ
       result[2] = a
       # 'access.kind != nkDotExpr' can happen for object constructors
       # which we don't check yet

compiler/ic/ic.nim

@@ -16,7 +16,7 @@ from std/os import removeFile, isAbsolute
 
 import ../../dist/checksums/src/checksums/sha1
 
-import ".." / nir / nirlineinfos
+import iclineinfos
 
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions, formatfloat]
@@ -59,8 +59,8 @@ type
     emittedTypeInfo*: seq[string]
     backendFlags*: set[ModuleBackendFlag]
 
-    syms*: seq[PackedSym]
-    types*: seq[PackedType]
+    syms*: OrderedTable[int32, PackedSym]
+    types*: OrderedTable[int32, PackedType]
     strings*: BiTable[string] # we could share these between modules.
     numbers*: BiTable[BiggestInt] # we also store floats in here so
                                   # that we can assure that every bit is kept
@@ -362,10 +362,10 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
   result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)
 
   if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item):
-    if t.uniqueId.item >= m.types.len:
-      setLen m.types, t.uniqueId.item+1
+    #if t.uniqueId.item >= m.types.len:
+    #  setLen m.types, t.uniqueId.item+1
 
-    var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
+    var p = PackedType(id: t.uniqueId.item, kind: t.kind, flags: t.flags, callConv: t.callConv,
       size: t.size, align: t.align, nonUniqueId: t.itemId.item,
       paddingAtEnd: t.paddingAtEnd)
     storeNode(p, t, n)
@@ -396,12 +396,12 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
   result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
 
   if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item):
-    if s.itemId.item >= m.syms.len:
-      setLen m.syms, s.itemId.item+1
+    #if s.itemId.item >= m.syms.len:
+    #  setLen m.syms, s.itemId.item+1
 
     assert sfForward notin s.flags
 
-    var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
+    var p = PackedSym(id: s.itemId.item, kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
       position: s.position, offset: s.offset, disamb: s.disamb, options: s.options,
       name: s.name.s.toLitId(m))
 
@@ -414,7 +414,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
       p.bitsize = s.bitsize
       p.alignment = s.alignment
 
-    p.externalName = toLitId(s.loc.r, m)
+    p.externalName = toLitId(s.loc.snippet, m)
     p.locFlags = s.loc.flags
     c.addMissing s.typ
     p.typ = s.typ.storeType(c, m)
@@ -613,6 +613,10 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
     f.loadSection section
     f.loadSeq data
 
+  template loadTableSection(section, data) {.dirty.} =
+    f.loadSection section
+    f.loadOrderedTable data
+
   template loadTabSection(section, data) {.dirty.} =
     f.loadSection section
     f.load data
@@ -645,8 +649,8 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
     loadTabSection topLevelSection, m.topLevel
 
     loadTabSection bodiesSection, m.bodies
-    loadSeqSection symsSection, m.syms
-    loadSeqSection typesSection, m.types
+    loadTableSection symsSection, m.syms
+    loadTableSection typesSection, m.types
 
     loadSeqSection typeInstCacheSection, m.typeInstCache
     loadSeqSection procInstCacheSection, m.procInstCache
@@ -691,6 +695,10 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
     f.storeSection section
     f.store data
 
+  template storeTableSection(section, data) {.dirty.} =
+    f.storeSection section
+    f.storeOrderedTable data
+
   storeTabSection stringsSection, m.strings
 
   storeSeqSection checkSumsSection, m.includes
@@ -714,9 +722,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeTabSection topLevelSection, m.topLevel
 
   storeTabSection bodiesSection, m.bodies
-  storeSeqSection symsSection, m.syms
+  storeTableSection symsSection, m.syms
 
-  storeSeqSection typesSection, m.types
+  storeTableSection typesSection, m.types
 
   storeSeqSection typeInstCacheSection, m.typeInstCache
   storeSeqSection procInstCacheSection, m.procInstCache
@@ -767,8 +775,8 @@ type
     status*: ModuleStatus
     symsInit, typesInit, loadedButAliveSetChanged*: bool
     fromDisk*: PackedModule
-    syms: seq[PSym] # indexed by itemId
-    types: seq[PType]
+    syms: OrderedTable[int32, PSym] # indexed by itemId
+    types: OrderedTable[int32, PType]
     module*: PSym # the one true module symbol.
     iface, ifaceHidden: Table[PIdent, seq[PackedItemId]]
       # PackedItemId so that it works with reexported symbols too
@@ -829,8 +837,8 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
     result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId])
   of nkSym:
     result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree[n].soperand))
-    if result.typ == nil and nfOpenSym notin result.flags:
-      result.typ = result.sym.typ
+    if result.typ == nil:
+      result.typ() = result.sym.typ
   of externIntLit:
     result.intVal = g[thisModule].fromDisk.numbers[n.litId]
   of nkStrLit..nkTripleStrLit:
@@ -843,8 +851,8 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
     assert n2.kind == nkNone
     transitionNoneToSym(result)
     result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree[n2].soperand))
-    if result.typ == nil and nfOpenSym notin result.flags:
-      result.typ = result.sym.typ
+    if result.typ == nil:
+      result.typ() = result.sym.typ
   else:
     for n0 in sonsReadonly(tree, n):
       result.addAllowNil loadNodes(c, g, thisModule, tree, n0)
@@ -934,10 +942,10 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     result.guard = loadSym(c, g, si, s.guard)
     result.bitsize = s.bitsize
     result.alignment = s.alignment
-  result.owner = loadSym(c, g, si, s.owner)
+  setOwner(result, loadSym(c, g, si, s.owner))
   let externalName = g[si].fromDisk.strings[s.externalName]
   if externalName != "":
-    result.loc.r = rope externalName
+    result.loc.snippet = externalName
   result.loc.flags = s.locFlags
   result.instantiatedFrom = loadSym(c, g, si, s.instantiatedFrom)
 
@@ -961,11 +969,11 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s:
         loadToReplayNodes(g, c.config, c.cache, m, g[int m])
 
     assert g[si].status in {loaded, storing, stored}
-    if not g[si].symsInit:
-      g[si].symsInit = true
-      setLen g[si].syms, g[si].fromDisk.syms.len
+    #if not g[si].symsInit:
+    #  g[si].symsInit = true
+    #  setLen g[si].syms, g[si].fromDisk.syms.len
 
-    if g[si].syms[s.item] == nil:
+    if g[si].syms.getOrDefault(s.item) == nil:
       if g[si].fromDisk.syms[s.item].kind != skModule:
         result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item)
         # store it here early on, so that recursions work properly:
@@ -990,7 +998,7 @@ proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
 proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                         t: PackedType; si, item: int32; result: PType) =
   result.sym = loadSym(c, g, si, t.sym)
-  result.owner = loadSym(c, g, si, t.owner)
+  setOwner(result, loadSym(c, g, si, t.owner))
   when false:
     for op, item in pairs t.attachedOps:
       result.attachedOps[op] = loadSym(c, g, si, item)
@@ -1012,11 +1020,11 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
     assert g[si].status in {loaded, storing, stored}
     assert t.item > 0
 
-    if not g[si].typesInit:
-      g[si].typesInit = true
-      setLen g[si].types, g[si].fromDisk.types.len
+    #if not g[si].typesInit:
+    #  g[si].typesInit = true
+    #  setLen g[si].types, g[si].fromDisk.types.len
 
-    if g[si].types[t.item] == nil:
+    if g[si].types.getOrDefault(t.item) == nil:
       result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item)
       # store it here early on, so that recursions work properly:
       g[si].types[t.item] = result
@@ -1054,7 +1062,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
                   name: getIdent(cache, splitFile(filename).name),
                   info: newLineInfo(fileIdx, 1, 1),
                   position: int(fileIdx))
-  m.module.owner = getPackage(conf, cache, fileIdx)
+  setOwner(m.module, getPackage(conf, cache, fileIdx))
   m.module.flags = m.fromDisk.moduleFlags
 
 proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
@@ -1155,10 +1163,7 @@ proc loadProcBody*(config: ConfigRef, cache: IdentCache;
 proc loadTypeFromId*(config: ConfigRef, cache: IdentCache;
                      g: var PackedModuleGraph; module: int; id: PackedItemId): PType =
   bench g.loadType:
-    if id.item < g[module].types.len:
-      result = g[module].types[id.item]
-    else:
-      result = nil
+    result = g[module].types.getOrDefault(id.item)
     if result == nil:
       var decoder = PackedDecoder(
         lastModule: int32(-1),
@@ -1171,10 +1176,7 @@ proc loadTypeFromId*(config: ConfigRef, cache: IdentCache;
 proc loadSymFromId*(config: ConfigRef, cache: IdentCache;
                     g: var PackedModuleGraph; module: int; id: PackedItemId): PSym =
   bench g.loadSym:
-    if id.item < g[module].syms.len:
-      result = g[module].syms[id.item]
-    else:
-      result = nil
+    result = g[module].syms.getOrDefault(id.item)
     if result == nil:
       var decoder = PackedDecoder(
         lastModule: int32(-1),
@@ -1190,19 +1192,6 @@ proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; confi
   else:
     ItemId(module: toFileIndex(id.module, g[thisModule].fromDisk, config).int32, item: id.item)
 
-proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) =
-  var bugs = 0
-  for i in 1 .. high(m.syms):
-    if m.syms[i].kind == skUnknown:
-      echo "EMPTY ID ", i, " module ", moduleId, " ", toFullPath(config, FileIndex(moduleId))
-      inc bugs
-  assert bugs == 0
-  when false:
-    var nones = 0
-    for i in 1 .. high(m.types):
-      inc nones, m.types[i].kind == tyNone
-    assert nones < 1
-
 proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                            moduleSym: PSym; m: PackedModule) =
   # For now only used for heavy debugging. In the future we could use this to reduce the

compiler/ic/iclineinfos.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
+#        (c) Copyright 2024 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.

compiler/ic/integrity.nim

@@ -10,7 +10,7 @@
 ## Integrity checking for a set of .rod files.
 ## The set must cover a complete Nim project.
 
-import std/sets
+import std/[sets, tables]
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -108,18 +108,18 @@ proc checkModule(c: var CheckedContext; m: PackedModule) =
   # We check that:
   # - Every symbol references existing types and symbols.
   # - Every tree node references existing types and symbols.
-  for i in 0..high(m.syms):
-    checkLocalSym c, int32(i)
+  for _, v in pairs(m.syms):
+    checkLocalSym c, v.id
 
   checkTree c, m.toReplay
   checkTree c, m.topLevel
 
   for e in m.exports:
-    assert e[1] >= 0 and e[1] < m.syms.len
+    #assert e[1] >= 0 and e[1] < m.syms.len
     assert e[0] == m.syms[e[1]].name
 
   for e in m.compilerProcs:
-    assert e[1] >= 0 and e[1] < m.syms.len
+    #assert e[1] >= 0 and e[1] < m.syms.len
     assert e[0] == m.syms[e[1]].name
 
   checkLocalSymIds c, m, m.converters

compiler/ic/navigator.nim

@@ -11,7 +11,7 @@
 ## IDE-like features. It uses the set of .rod files to accomplish
 ## its task. The set must cover a complete Nim project.
 
-import std/sets
+import std/[sets, tables]
 
 from std/os import nil
 from std/private/miscdollars import toLocation
@@ -20,7 +20,7 @@ when defined(nimPreviewSlimSystem):
   import std/assertions
 
 import ".." / [ast, modulegraphs, msgs, options]
-import ".." / nir / nirlineinfos
+import iclineinfos
 import packed_ast, bitabs, ic
 
 type

compiler/ic/packed_ast.nim

@@ -16,7 +16,7 @@ import std/[hashes, tables, strtabs]
 import bitabs, rodfiles
 import ".." / [ast, options]
 
-import ".." / nir / nirlineinfos
+import iclineinfos
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -47,6 +47,7 @@ type
     path*: NodeId
 
   PackedSym* = object
+    id*: int32
     kind*: TSymKind
     name*: LitId
     typ*: PackedItemId
@@ -71,6 +72,7 @@ type
     instantiatedFrom*: PackedItemId
 
   PackedType* = object
+    id*: int32
     kind*: TTypeKind
     callConv*: TCallingConvention
     #nodekind*: TNodeKind

compiler/ic/rodfiles.nim

@@ -19,6 +19,8 @@ from std/typetraits import supportsCopyMem
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions]
 
+import std / tables
+
 ## Overview
 ## ========
 ## `RodFile` represents a Rod File (versioned binary format), and the
@@ -170,6 +172,18 @@ proc storeSeq*[T](f: var RodFile; s: seq[T]) =
     for i in 0..<s.len:
       storePrim(f, s[i])
 
+proc storeOrderedTable*[K, T](f: var RodFile; s: OrderedTable[K, T]) =
+  if f.err != ok: return
+  if s.len >= high(int32):
+    setError f, tooBig
+    return
+  var lenPrefix = int32(s.len)
+  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    setError f, ioFailure
+  else:
+    for _, v in s:
+      storePrim(f, v)
+
 proc loadPrim*(f: var RodFile; s: var string) =
   ## Read a string, the length was stored as a prefix
   if f.err != ok: return
@@ -211,6 +225,19 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
     for i in 0..<lenPrefix:
       loadPrim(f, s[i])
 
+proc loadOrderedTable*[K, T](f: var RodFile; s: var OrderedTable[K, T]) =
+  ## `T` must be compatible with `copyMem`, see `loadPrim`
+  if f.err != ok: return
+  var lenPrefix = int32(0)
+  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    setError f, ioFailure
+  else:
+    s = initOrderedTable[K, T](lenPrefix)
+    for i in 0..<lenPrefix:
+      var x = default T
+      loadPrim(f, x)
+      s[x.id] = x
+
 proc storeHeader*(f: var RodFile; cookie = defaultCookie) =
   ## stores the header which is described by `cookie`.
   if f.err != ok: return

compiler/importer.nim

@@ -246,7 +246,8 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool)
     result = createModuleAliasImpl(realModule.name)
   if importHidden:
     result.options.incl optImportHidden
-  c.unusedImports.add((result, n.info))
+  let moduleIdent = if n.kind == nkInfix: n[^1] else: n
+  c.unusedImports.add((result, moduleIdent.info))
   c.importModuleMap[result.id] = realModule.id
   c.importModuleLookup.mgetOrPut(result.name.id, @[]).addUnique realModule.id
 

compiler/injectdestructors.nim

@@ -317,8 +317,9 @@ proc isCriticalLink(dest: PNode): bool {.inline.} =
   ]#
   result = dest.kind != nkSym
 
-proc finishCopy(c: var Con; result, dest: PNode; isFromSink: bool) =
-  if c.graph.config.selectedGC == gcOrc:
+proc finishCopy(c: var Con; result, dest: PNode; flags: set[MoveOrCopyFlag]; isFromSink: bool) =
+  if c.graph.config.selectedGC == gcOrc and IsExplicitSink notin flags:
+    # add cyclic flag, but not to sink calls, which IsExplicitSink generates
     let t = dest.typ.skipTypes(tyUserTypeClasses + {tyGenericInst, tyAlias, tySink, tyDistinct})
     if cyclicType(c.graph, t):
       result.add boolLit(c.graph, result.info, isFromSink or isCriticalLink(dest))
@@ -331,7 +332,7 @@ proc genMarkCyclic(c: var Con; result, dest: PNode) =
         result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, dest)
       else:
         let xenv = genBuiltin(c.graph, c.idgen, mAccessEnv, "accessEnv", dest)
-        xenv.typ = getSysType(c.graph, dest.info, tyPointer)
+        xenv.typ() = getSysType(c.graph, dest.info, tyPointer)
         result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, xenv)
 
 proc genCopyNoCheck(c: var Con; dest, ri: PNode; a: TTypeAttachedOp): PNode =
@@ -407,12 +408,15 @@ proc genWasMoved(c: var Con, n: PNode): PNode =
 proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
   result = newNodeI(nkCall, info)
   result.add(newSymNode(createMagic(c.graph, c.idgen, "default", mDefault)))
-  result.typ = t
+  result.typ() = t
 
 proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
   # generate: (let tmp = v; reset(v); tmp)
   if (not hasDestructor(c, n.typ)) and c.inEnsureMove == 0:
-    assert n.kind != nkSym or not hasDestructor(c, n.sym.typ)
+    assert n.kind != nkSym or not hasDestructor(c, n.sym.typ) or
+          (n.typ.kind == tyPtr and n.sym.typ.kind == tyRef)
+      # bug #23505; transformed by `transf`: addr (deref ref) -> ptr
+      # we know it's really a pointer; so here we assign it directly
     result = copyTree(n)
   else:
     result = newNodeIT(nkStmtListExpr, n.info, n.typ)
@@ -461,7 +465,7 @@ proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode =
       var newCall = newTreeIT(nkCall, src.info, src.typ,
             newSymNode(op),
             src)
-      c.finishCopy(newCall, n, isFromSink = true)
+      c.finishCopy(newCall, n, {}, isFromSink = true)
       result.add newTreeI(nkFastAsgn,
           src.info, tmp,
           newCall
@@ -470,7 +474,7 @@ proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode =
       result.add c.genWasMoved(tmp)
       var m = c.genCopy(tmp, n, {})
       m.add p(n, c, s, normal)
-      c.finishCopy(m, n, isFromSink = true)
+      c.finishCopy(m, n, {}, isFromSink = true)
       result.add m
     if isLValue(n) and not isCapturedVar(n) and nTyp.skipTypes(abstractInst).kind != tyRef and c.inSpawn == 0:
       message(c.graph.config, n.info, hintPerformance,
@@ -498,7 +502,7 @@ proc containsConstSeq(n: PNode): bool =
     return true
   result = false
   case n.kind
-  of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv:
+  of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv, nkCast:
     result = containsConstSeq(n[1])
   of nkObjConstr, nkClosure:
     for i in 1..<n.len:
@@ -650,7 +654,7 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false,
       for j in 0 ..< it.len-1:
         branch[j] = copyTree(it[j])
       var ofScope = nestedScope(s, it.lastSon)
-      branch[^1] = if it[^1].typ.isEmptyType or willProduceStmt:
+      branch[^1] = if n.typ.isEmptyType or it[^1].typ.isEmptyType or willProduceStmt:
                      processScope(c, ofScope, maybeVoid(it[^1], ofScope))
                    else:
                      processScopeExpr(c, ofScope, it[^1], processCall, tmpFlags)
@@ -698,7 +702,7 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false,
         #Condition needs to be destroyed outside of the condition/branch scope
         branch[0] = p(it[0], c, s, normal)
 
-      branch[^1] = if it[^1].typ.isEmptyType or willProduceStmt:
+      branch[^1] = if n.typ.isEmptyType or it[^1].typ.isEmptyType or willProduceStmt:
                      processScope(c, branchScope, maybeVoid(it[^1], branchScope))
                    else:
                      processScopeExpr(c, branchScope, it[^1], processCall, tmpFlags)
@@ -758,7 +762,7 @@ proc pRaiseStmt(n: PNode, c: var Con; s: var Scope): PNode =
       let tmp = c.getTemp(s, n[0].typ, n.info)
       var m = c.genCopyNoCheck(tmp, n[0], attachedAsgn)
       m.add p(n[0], c, s, normal)
-      c.finishCopy(m, n[0], isFromSink = false)
+      c.finishCopy(m, n[0], {}, isFromSink = false)
       result = newTree(nkStmtList, c.genWasMoved(tmp), m)
       var toDisarm = n[0]
       if toDisarm.kind == nkStmtListExpr: toDisarm = toDisarm.lastSon
@@ -818,14 +822,17 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
           n[1].typ.skipTypes(abstractInst-{tyOwned}).kind == tyOwned:
         # allow conversions from owned to unowned via this little hack:
         let nTyp = n[1].typ
-        n[1].typ = n.typ
+        n[1].typ() = n.typ
         result[1] = p(n[1], c, s, sinkArg)
-        result[1].typ = nTyp
+        result[1].typ() = nTyp
       else:
         result[1] = p(n[1], c, s, sinkArg)
     elif n.kind in {nkObjDownConv, nkObjUpConv}:
       result = copyTree(n)
       result[0] = p(n[0], c, s, sinkArg)
+    elif n.kind == nkCast and n.typ.skipTypes(abstractInst).kind in {tyString, tySequence}:
+      result = copyTree(n)
+      result[1] = p(n[1], c, s, sinkArg)
     elif n.typ == nil:
       # 'raise X' can be part of a 'case' expression. Deal with it here:
       result = p(n, c, s, normal)
@@ -869,7 +876,8 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
       for i in 1..<n.len:
         if n[i].kind == nkExprColonExpr:
           let field = lookupFieldAgain(t, n[i][0].sym)
-          if field != nil and sfCursor in field.flags:
+          if field != nil and (sfCursor in field.flags or field.typ.kind in {tyOpenArray, tyVarargs}):
+            # don't sink fields with openarray types
             result[i][1] = p(n[i][1], c, s, normal)
           else:
             result[i][1] = p(n[i][1], c, s, m)
@@ -890,7 +898,8 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
       elif c.inSpawn > 0:
         c.inSpawn.dec
 
-      let parameters = n[0].typ
+      # bug #23907; skips tyGenericInst for generic callbacks
+      let parameters = if n[0].typ != nil: n[0].typ.skipTypes(abstractInst) else: n[0].typ
       let L = if parameters != nil: parameters.signatureLen else: 0
 
       when false:
@@ -1013,9 +1022,9 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
           n[1].typ.skipTypes(abstractInst-{tyOwned}).kind == tyOwned:
         # allow conversions from owned to unowned via this little hack:
         let nTyp = n[1].typ
-        n[1].typ = n.typ
+        n[1].typ() = n.typ
         result[1] = p(n[1], c, s, mode)
-        result[1].typ = nTyp
+        result[1].typ() = nTyp
       else:
         result[1] = p(n[1], c, s, mode)
 
@@ -1165,7 +1174,7 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopy
         result = c.genCopy(dest, ri, flags)
         dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
-        c.finishCopy(result, dest, isFromSink = false)
+        c.finishCopy(result, dest, flags, isFromSink = false)
     of nkBracket:
       # array constructor
       if ri.len > 0 and isDangerousSeq(ri.typ):
@@ -1173,7 +1182,7 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopy
         result = c.genCopy(dest, ri, flags)
         dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
-        c.finishCopy(result, dest, isFromSink = false)
+        c.finishCopy(result, dest, flags, isFromSink = false)
       else:
         result = c.genSink(s, dest, p(ri, c, s, consumed), flags)
     of nkObjConstr, nkTupleConstr, nkClosure, nkCharLit..nkNilLit:
@@ -1183,9 +1192,9 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopy
         # Rule 3: `=sink`(x, z); wasMoved(z)
         let snk = c.genSink(s, dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
-      elif ri.sym.kind != skParam and ri.sym.owner == c.owner and
-          isLastRead(ri, c, s) and canBeMoved(c, dest.typ) and not isCursor(ri) and
-          not ({sfGlobal, sfPure} <= ri.sym.flags):
+      elif ri.sym.kind != skParam and
+          isAnalysableFieldAccess(ri, c.owner) and
+          isLastRead(ri, c, s) and canBeMoved(c, dest.typ):
         # Rule 3: `=sink`(x, z); wasMoved(z)
         let snk = c.genSink(s, dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
@@ -1194,7 +1203,7 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopy
         result = c.genCopy(dest, ri, flags)
         dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
-        c.finishCopy(result, dest, isFromSink = false)
+        c.finishCopy(result, dest, flags, isFromSink = false)
     of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv, nkCast:
       result = c.genSink(s, dest, p(ri, c, s, sinkArg), flags)
     of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
@@ -1214,7 +1223,7 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopy
         result = c.genCopy(dest, ri, flags)
         dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
-        c.finishCopy(result, dest, isFromSink = false)
+        c.finishCopy(result, dest, flags, isFromSink = false)
 
 when false:
   proc computeUninit(c: var Con) =

compiler/installer.ini

@@ -6,11 +6,11 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64;loongarch64
   macosx: i386;amd64;powerpc64;arm64
   solaris: i386;amd64;sparc;sparc64
   freebsd: i386;amd64;powerpc64;arm;arm64;riscv64;sparc64;mips;mipsel;mips64;mips64el;powerpc;powerpc64el
-  netbsd: i386;amd64
+  netbsd: i386;amd64;arm64
   openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
   crossos: amd64

compiler/jsgen.nim

@@ -33,7 +33,7 @@ import
   nversion, msgs, idents, types,
   ropes, wordrecg, renderer,
   cgmeth, lowerings, sighashes, modulegraphs, lineinfos,
-  transf, injectdestructors, sourcemap, astmsgs, backendpragmas,
+  transf, injectdestructors, sourcemap, astmsgs, pushpoppragmas,
   mangleutils
 
 import pipelineutils
@@ -103,22 +103,33 @@ type
     prc: PSym
     globals, locals, body: Rope
     options: TOptions
-    optionsStack: seq[TOptions]
+    optionsStack: seq[(TOptions, TNoteKinds)]
     module: BModule
     g: PGlobals
     beforeRetNeeded: bool
     unique: int    # for temp identifier generation
     blocks: seq[TBlock]
     extraIndent: int
-    declaredGlobals: IntSet
     previousFileName: string  # For frameInfo inside templates.
+    # legacy: generatedParamCopies and up fields are used for jsNoLambdaLifting
+    generatedParamCopies: IntSet
+    up: PProc     # up the call chain; required for closure support
 
 template config*(p: PProc): ConfigRef = p.module.config
 
 proc indentLine(p: PProc, r: Rope): Rope =
   var p = p
-  let ind = p.blocks.len + p.extraIndent
-  result = repeat(' ', ind*2) & r
+  if jsNoLambdaLifting in p.config.legacyFeatures:
+    var ind = 0
+    while true:
+      inc ind, p.blocks.len + p.extraIndent
+      if p.up == nil or p.up.prc != p.prc.owner:
+        break
+      p = p.up
+    result = repeat(' ', ind*2) & r
+  else:
+    let ind = p.blocks.len + p.extraIndent
+    result = repeat(' ', ind*2) & r
 
 template line(p: PProc, added: string) =
   p.body.add(indentLine(p, rope(added)))
@@ -169,10 +180,6 @@ proc initProcOptions(module: BModule): TOptions =
 proc newInitProc(globals: PGlobals, module: BModule): PProc =
   result = newProc(globals, module, nil, initProcOptions(module))
 
-proc declareGlobal(p: PProc; id: int; r: Rope) =
-  if p.prc != nil and not p.declaredGlobals.containsOrIncl(id):
-    p.locals.addf("global $1;$n", [r])
-
 const
   MappedToObject = {tyObject, tyArray, tyTuple, tyOpenArray,
     tySet, tyVarargs}
@@ -188,7 +195,7 @@ proc mapType(typ: PType): TJSTypeKind =
   of tyPointer:
     # treat a tyPointer like a typed pointer to an array of bytes
     result = etyBaseIndex
-  of tyRange, tyDistinct, tyOrdinal, tyProxy, tyLent:
+  of tyRange, tyDistinct, tyOrdinal, tyError, tyLent:
     # tyLent is no-op as JS has pass-by-reference semantics
     result = mapType(skipModifier t)
   of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar: result = etyInt
@@ -239,7 +246,7 @@ proc mangleName(m: BModule, s: PSym): Rope =
     for chr in name:
       if chr notin {'A'..'Z','a'..'z','_','$','0'..'9'}:
         return false
-  result = s.loc.r
+  result = s.loc.snippet
   if result == "":
     if s.kind == skField and s.name.s.validJsName:
       result = rope(s.name.s)
@@ -270,7 +277,7 @@ proc mangleName(m: BModule, s: PSym): Rope =
       else:
         result.add("_")
         result.add(rope(s.id))
-    s.loc.r = result
+    s.loc.snippet = result
 
 proc escapeJSString(s: string): string =
   result = newStringOfCap(s.len + s.len shr 2)
@@ -604,6 +611,17 @@ template unaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   r.res = frmt % [a, tmp]
   r.kind = resExpr
 
+proc genBreakState(p: PProc, n: PNode, r: var TCompRes) =
+  var a: TCompRes = default(TCompRes)
+  # mangle `:state` properly somehow
+  if n.kind == nkClosure:
+    gen(p, n[1], a)
+    r.res = "(($1).HEX3Astate < 0)" % [rdLoc(a)]
+  else:
+    gen(p, n, a)
+    r.res = "((($1.ClE_0).HEX3Astate) < 0)" % [rdLoc(a)]
+  r.kind = resExpr
+
 proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   var
     x, y: TCompRes = default(TCompRes)
@@ -974,7 +992,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
         # if isJsObject(throwObj.typ):
         if isImportedException(throwObj.typ, p.config):
           orExpr.addf("lastJSError instanceof $1",
-            [throwObj.typ.sym.loc.r])
+            [throwObj.typ.sym.loc.snippet])
         else:
           orExpr.addf("isObj(lastJSError.m_type, $1)",
                [genTypeInfo(p, throwObj.typ)])
@@ -984,8 +1002,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
       # If some branch requires a local alias introduce it here. This is needed
       # since JS cannot do ``catch x as y``.
       if excAlias != nil:
-        excAlias.sym.loc.r = mangleName(p.module, excAlias.sym)
-        lineF(p, "var $1 = lastJSError;$n", excAlias.sym.loc.r)
+        excAlias.sym.loc.snippet = mangleName(p.module, excAlias.sym)
+        lineF(p, "var $1 = lastJSError;$n", excAlias.sym.loc.snippet)
       gen(p, n[i][^1], a)
       moveInto(p, a, r)
       lineF(p, "}$n", [])
@@ -1023,7 +1041,7 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
     a, b, cond, stmt: TCompRes = default(TCompRes)
   genLineDir(p, n)
   gen(p, n[0], cond)
-  let typeKind = skipTypes(n[0].typ, abstractVar).kind
+  let typeKind = skipTypes(n[0].typ, abstractVar+{tyRange}).kind
   var transferRange = false
   let anyString = typeKind in {tyString, tyCstring}
   case typeKind
@@ -1091,14 +1109,19 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
           lineF(p, "break;$n", [])
     of nkElse:
       if transferRange:
-         lineF(p, "else{$n", [])
+        if n.len == 2: # a dangling else for a case statement
+          transferRange = false
+          lineF(p, "switch ($1) {$n", [cond.rdLoc])
+          lineF(p, "default: $n", [])
+        else:
+          lineF(p, "else{$n", [])
       else:
         lineF(p, "default: $n", [])
       p.nested:
         gen(p, it[0], stmt)
         moveInto(p, stmt, r)
         if transferRange:
-           lineF(p, "}$n", [])
+          lineF(p, "}$n", [])
         else:
           lineF(p, "break;$n", [])
     else: internalError(p.config, it.info, "jsgen.genCaseStmt")
@@ -1205,12 +1228,13 @@ proc genIf(p: PProc, n: PNode, r: var TCompRes) =
 proc generateHeader(p: PProc, prc: PSym): Rope =
   result = ""
   let typ = prc.typ
-  if typ.callConv == ccClosure:
-    # we treat Env as the `this` parameter of the function
-    # to keep it simple
-    let env = prc.ast[paramsPos].lastSon
-    assert env.kind == nkSym, "env is missing"
-    env.sym.loc.r = "this"
+  if jsNoLambdaLifting notin p.config.legacyFeatures:
+    if typ.callConv == ccClosure:
+      # we treat Env as the `this` parameter of the function
+      # to keep it simple
+      let env = prc.ast[paramsPos].lastSon
+      assert env.kind == nkSym, "env is missing"
+      env.sym.loc.snippet = "this"
 
   for i in 1..<typ.n.len:
     assert(typ.n[i].kind == nkSym)
@@ -1244,9 +1268,10 @@ const
 
 proc needsNoCopy(p: PProc; y: PNode): bool =
   return y.kind in nodeKindsNeedNoCopy or
-        ((mapType(y.typ) != etyBaseIndex) and
+        ((mapType(y.typ) != etyBaseIndex or
+          (jsNoLambdaLifting in p.config.legacyFeatures and y.kind == nkSym and y.sym.kind == skParam)) and
           (skipTypes(y.typ, abstractInst).kind in
-            {tyRef, tyPtr, tyLent, tyVar, tyCstring, tyProc, tyOwned} + IntegralTypes))
+            {tyRef, tyPtr, tyLent, tyVar, tyCstring, tyProc, tyOwned, tyOpenArray} + IntegralTypes))
 
 proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
   var a, b: TCompRes = default(TCompRes)
@@ -1273,7 +1298,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
       lineF(p, "$1 = nimCopy(null, $2, $3);$n",
                [a.rdLoc, b.res, genTypeInfo(p, y.typ)])
   of etyObject:
-    if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind in {tyVar, tyLent, tyOpenArray, tyVarargs} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
@@ -1351,8 +1376,8 @@ proc genFieldAddr(p: PProc, n: PNode, r: var TCompRes) =
   else:
     if b[1].kind != nkSym: internalError(p.config, b[1].info, "genFieldAddr")
     var f = b[1].sym
-    if f.loc.r == "": f.loc.r = mangleName(p.module, f)
-    r.res = makeJSString($f.loc.r)
+    if f.loc.snippet == "": f.loc.snippet = mangleName(p.module, f)
+    r.res = makeJSString($f.loc.snippet)
   internalAssert p.config, a.typ != etyBaseIndex
   r.address = a.res
   r.kind = resExpr
@@ -1379,8 +1404,8 @@ proc genFieldAccess(p: PProc, n: PNode, r: var TCompRes) =
   else:
     if n[1].kind != nkSym: internalError(p.config, n[1].info, "genFieldAccess")
     var f = n[1].sym
-    if f.loc.r == "": f.loc.r = mangleName(p.module, f)
-    r.res = "$1.$2" % [r.res, f.loc.r]
+    if f.loc.snippet == "": f.loc.snippet = mangleName(p.module, f)
+    r.res = "$1.$2" % [r.res, f.loc.snippet]
     mkTemp(1)
   r.kind = resExpr
 
@@ -1400,11 +1425,11 @@ proc genCheckedFieldOp(p: PProc, n: PNode, addrTyp: PType, r: var TCompRes) =
   # Field symbol
   var field = accessExpr[1].sym
   internalAssert p.config, field.kind == skField
-  if field.loc.r == "": field.loc.r = mangleName(p.module, field)
+  if field.loc.snippet == "": field.loc.snippet = mangleName(p.module, field)
   # Discriminant symbol
   let disc = checkExpr[2].sym
   internalAssert p.config, disc.kind == skField
-  if disc.loc.r == "": disc.loc.r = mangleName(p.module, disc)
+  if disc.loc.snippet == "": disc.loc.snippet = mangleName(p.module, disc)
 
   var setx: TCompRes = default(TCompRes)
   gen(p, checkExpr[1], setx)
@@ -1421,16 +1446,16 @@ proc genCheckedFieldOp(p: PProc, n: PNode, addrTyp: PType, r: var TCompRes) =
   useMagic(p, "reprDiscriminant") # no need to offset by firstOrd unlike for cgen
   let msg = genFieldDefect(p.config, field.name.s, disc)
   lineF(p, "if ($1[$2.$3]$4undefined) { raiseFieldError2(makeNimstrLit($5), reprDiscriminant($2.$3, $6)); }$n",
-    setx.res, tmp, disc.loc.r, if negCheck: "!==" else: "===",
+    setx.res, tmp, disc.loc.snippet, if negCheck: "!==" else: "===",
     makeJSString(msg), genTypeInfo(p, disc.typ))
 
   if addrTyp != nil and mapType(p, addrTyp) == etyBaseIndex:
     r.typ = etyBaseIndex
-    r.res = makeJSString($field.loc.r)
+    r.res = makeJSString($field.loc.snippet)
     r.address = tmp
   else:
     r.typ = etyNone
-    r.res = "$1.$2" % [tmp, field.loc.r]
+    r.res = "$1.$2" % [tmp, field.loc.snippet]
   r.kind = resExpr
 
 proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
@@ -1461,7 +1486,7 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
   r.kind = resExpr
 
 proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) =
-  var ty = skipTypes(n[0].typ, abstractVarRange)
+  var ty = skipTypes(n[0].typ, abstractVarRange+tyUserTypeClasses)
   if ty.kind in {tyRef, tyPtr, tyLent, tyOwned}: ty = skipTypes(ty.elementType, abstractVarRange)
   case ty.kind
   of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs:
@@ -1497,10 +1522,10 @@ template isIndirect(x: PSym): bool =
 
 proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) =
   let s = n.sym
-  if s.loc.r == "": internalError(p.config, n.info, "genAddr: 3")
+  if s.loc.snippet == "": internalError(p.config, n.info, "genAddr: 3")
   case s.kind
   of skParam:
-    r.res = s.loc.r
+    r.res = s.loc.snippet
     r.address = ""
     r.typ = etyNone
   of skVar, skLet, skResult:
@@ -1514,15 +1539,15 @@ proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) =
       # make addr() a no-op:
       r.typ = etyNone
       if isIndirect(s):
-        r.res = s.loc.r & "[0]"
+        r.res = s.loc.snippet & "[0]"
       else:
-        r.res = s.loc.r
+        r.res = s.loc.snippet
       r.address = ""
     elif {sfGlobal, sfAddrTaken} * s.flags != {} or jsType == etyBaseIndex:
       # for ease of code generation, we do not distinguish between
       # sfAddrTaken and sfGlobal.
       r.typ = etyBaseIndex
-      r.address = s.loc.r
+      r.address = s.loc.snippet
       r.res = rope("0")
     else:
       # 'var openArray' for instance produces an 'addr' but this is harmless:
@@ -1560,15 +1585,8 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
         else: internalError(p.config, n[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')')
     of nkObjDownConv:
       gen(p, n[0], r)
-    of nkHiddenDeref:
+    of nkHiddenDeref, nkDerefExpr:
       gen(p, n[0], r)
-    of nkDerefExpr:
-      var x = n[0]
-      if n.kind == nkHiddenAddr:
-        x = n[0][0]
-        if n.typ.skipTypes(abstractVar).kind != tyOpenArray:
-          x.typ = n.typ
-      gen(p, x, r)
     of nkHiddenAddr:
       gen(p, n[0], r)
     of nkConv:
@@ -1595,7 +1613,30 @@ proc attachProc(p: PProc; s: PSym) =
 
 proc genProcForSymIfNeeded(p: PProc, s: PSym) =
   if not p.g.generatedSyms.containsOrIncl(s.id):
-    attachProc(p, s)
+    if jsNoLambdaLifting in p.config.legacyFeatures:
+      let newp = genProc(p, s)
+      var owner = p
+      while owner != nil and owner.prc != s.owner:
+        owner = owner.up
+      if owner != nil: owner.locals.add(newp)
+      else: attachProc(p, newp, s)
+    else:
+      attachProc(p, s)
+
+proc genCopyForParamIfNeeded(p: PProc, n: PNode) =
+  let s = n.sym
+  if p.prc == s.owner or needsNoCopy(p, n):
+    return
+  var owner = p.up
+  while true:
+    if owner == nil:
+      internalError(p.config, n.info, "couldn't find the owner proc of the closed over param: " & s.name.s)
+    if owner.prc == s.owner:
+      if not owner.generatedParamCopies.containsOrIncl(s.id):
+        let copy = "$1 = nimCopy(null, $1, $2);$n" % [s.loc.snippet, genTypeInfo(p, s.typ)]
+        owner.locals.add(owner.indentLine(copy))
+      return
+    owner = owner.up
 
 proc genVarInit(p: PProc, v: PSym, n: PNode)
 
@@ -1603,38 +1644,40 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
   var s = n.sym
   case s.kind
   of skVar, skLet, skParam, skTemp, skResult, skForVar:
-    if s.loc.r == "":
+    if s.loc.snippet == "":
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
     if sfCompileTime in s.flags:
       genVarInit(p, s, if s.astdef != nil: s.astdef else: newNodeI(nkEmpty, s.info))
+    if jsNoLambdaLifting in p.config.legacyFeatures and s.kind == skParam:
+      genCopyForParamIfNeeded(p, n)
     let k = mapType(p, s.typ)
     if k == etyBaseIndex:
       r.typ = etyBaseIndex
       if {sfAddrTaken, sfGlobal} * s.flags != {}:
         if isIndirect(s):
-          r.address = "$1[0][0]" % [s.loc.r]
-          r.res = "$1[0][1]" % [s.loc.r]
+          r.address = "$1[0][0]" % [s.loc.snippet]
+          r.res = "$1[0][1]" % [s.loc.snippet]
         else:
-          r.address = "$1[0]" % [s.loc.r]
-          r.res = "$1[1]" % [s.loc.r]
+          r.address = "$1[0]" % [s.loc.snippet]
+          r.res = "$1[1]" % [s.loc.snippet]
       else:
-        r.address = s.loc.r
-        r.res = s.loc.r & "_Idx"
+        r.address = s.loc.snippet
+        r.res = s.loc.snippet & "_Idx"
     elif isIndirect(s):
-      r.res = "$1[0]" % [s.loc.r]
+      r.res = "$1[0]" % [s.loc.snippet]
     else:
-      r.res = s.loc.r
+      r.res = s.loc.snippet
   of skConst:
     genConstant(p, s)
-    if s.loc.r == "":
+    if s.loc.snippet == "":
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
-    r.res = s.loc.r
+    r.res = s.loc.snippet
   of skProc, skFunc, skConverter, skMethod, skIterator:
     if sfCompileTime in s.flags:
       localError(p.config, n.info, "request to generate code for .compileTime proc: " &
           s.name.s)
     discard mangleName(p.module, s)
-    r.res = s.loc.r
+    r.res = s.loc.snippet
     if lfNoDecl in s.loc.flags or s.magic notin generatedMagics or
        {sfImportc, sfInfixCall} * s.flags != {}:
       discard
@@ -1646,13 +1689,13 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
     else:
       genProcForSymIfNeeded(p, s)
   else:
-    if s.loc.r == "":
+    if s.loc.snippet == "":
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
     if mapType(p, s.typ) == etyBaseIndex:
-      r.address = s.loc.r
-      r.res = s.loc.r & "_Idx"
+      r.address = s.loc.snippet
+      r.res = s.loc.snippet & "_Idx"
     else:
-      r.res = s.loc.r
+      r.res = s.loc.snippet
   r.kind = resVal
 
 proc genDeref(p: PProc, n: PNode, r: var TCompRes) =
@@ -1794,9 +1837,9 @@ proc genPatternCall(p: PProc; n: PNode; pat: string; typ: PType;
 proc genInfixCall(p: PProc, n: PNode, r: var TCompRes) =
   # don't call '$' here for efficiency:
   let f = n[0].sym
-  if f.loc.r == "": f.loc.r = mangleName(p.module, f)
+  if f.loc.snippet == "": f.loc.snippet = mangleName(p.module, f)
   if sfInfixCall in f.flags:
-    let pat = $n[0].sym.loc.r
+    let pat = $n[0].sym.loc.snippet
     internalAssert p.config, pat.len > 0
     if pat.contains({'#', '(', '@'}):
       var typ = skipTypes(n[0].typ, abstractInst)
@@ -1911,7 +1954,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   of tyInt8..tyInt32, tyUInt8..tyUInt32, tyEnum, tyChar:
     result = putToSeq("0", indirect)
   of tyInt, tyUInt:
-    if $t.sym.loc.r == "bigint":
+    if $t.sym.loc.snippet == "bigint":
       result = putToSeq("0n", indirect)
     else:
       result = putToSeq("0", indirect)
@@ -1971,7 +2014,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
       result = putToSeq("null", indirect)
   of tySequence, tyString:
     result = putToSeq("[]", indirect)
-  of tyCstring, tyProc:
+  of tyCstring, tyProc, tyOpenArray:
     result = putToSeq("null", indirect)
   of tyStatic:
     if t.n != nil:
@@ -2023,7 +2066,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     gen(p, n, a)
     case mapType(p, v.typ)
     of etyObject, etySeq:
-      if needsNoCopy(p, n):
+      if v.typ.kind in {tyOpenArray, tyVarargs} or needsNoCopy(p, n):
         s = a.res
       else:
         useMagic(p, "nimCopy")
@@ -2033,28 +2076,28 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       if a.typ == etyBaseIndex:
         if targetBaseIndex:
           line(p, runtimeFormat(varCode & " = $3, $2_Idx = $4;$n",
-                   [returnType, v.loc.r, a.address, a.res]))
+                   [returnType, v.loc.snippet, a.address, a.res]))
         else:
           if isIndirect(v):
             line(p, runtimeFormat(varCode & " = [[$3, $4]];$n",
-                     [returnType, v.loc.r, a.address, a.res]))
+                     [returnType, v.loc.snippet, a.address, a.res]))
           else:
             line(p, runtimeFormat(varCode & " = [$3, $4];$n",
-                     [returnType, v.loc.r, a.address, a.res]))
+                     [returnType, v.loc.snippet, a.address, a.res]))
       else:
         if targetBaseIndex:
           let tmp = p.getTemp
           lineF(p, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n",
-                   [tmp, a.res, v.loc.r])
+                   [tmp, a.res, v.loc.snippet])
         else:
-          line(p, runtimeFormat(varCode & " = $3;$n", [returnType, v.loc.r, a.res]))
+          line(p, runtimeFormat(varCode & " = $3;$n", [returnType, v.loc.snippet, a.res]))
       return
     else:
       s = a.res
     if isIndirect(v):
-      line(p, runtimeFormat(varCode & " = [$3];$n", [returnType, v.loc.r, s]))
+      line(p, runtimeFormat(varCode & " = [$3];$n", [returnType, v.loc.snippet, s]))
     else:
-      line(p, runtimeFormat(varCode & " = $3;$n", [returnType, v.loc.r, s]))
+      line(p, runtimeFormat(varCode & " = $3;$n", [returnType, v.loc.snippet, s]))
 
   if useReloadingGuard or useGlobalPragmas:
     dec p.extraIndent
@@ -2137,20 +2180,20 @@ proc genConStrStr(p: PProc, n: PNode, r: var TCompRes) =
   if skipTypes(n[1].typ, abstractVarRange).kind == tyChar:
     r.res.add("[$1].concat(" % [a.res])
   else:
-    r.res.add("($1 || []).concat(" % [a.res])
+    r.res.add("($1).concat(" % [a.res])
 
   for i in 2..<n.len - 1:
     gen(p, n[i], a)
     if skipTypes(n[i].typ, abstractVarRange).kind == tyChar:
       r.res.add("[$1]," % [a.res])
     else:
-      r.res.add("$1 || []," % [a.res])
+      r.res.add("$1," % [a.res])
 
   gen(p, n[^1], a)
   if skipTypes(n[^1].typ, abstractVarRange).kind == tyChar:
     r.res.add("[$1])" % [a.res])
   else:
-    r.res.add("$1 || [])" % [a.res])
+    r.res.add("$1)" % [a.res])
 
 proc genReprAux(p: PProc, n: PNode, r: var TCompRes, magic: string, typ: Rope = "") =
   useMagic(p, magic)
@@ -2217,16 +2260,17 @@ proc genDefault(p: PProc, n: PNode; r: var TCompRes) =
   r.res = createVar(p, n.typ, indirect = false)
   r.kind = resExpr
 
-proc genReset(p: PProc, n: PNode) =
+proc genWasMoved(p: PProc, n: PNode) =
+  # TODO: it should be done by nir
   var x: TCompRes = default(TCompRes)
-  useMagic(p, "genericReset")
   gen(p, n[1], x)
   if x.typ == etyBaseIndex:
     lineF(p, "$1 = null, $2 = 0;$n", [x.address, x.res])
   else:
-    let (a, tmp) = maybeMakeTempAssignable(p, n[1], x)
-    lineF(p, "$1 = genericReset($3, $2);$n", [a,
-                  genTypeInfo(p, n[1].typ), tmp])
+    var y: TCompRes = default(TCompRes)
+    genDefault(p, n[1], y)
+    let (a, _) = maybeMakeTempAssignable(p, n[1], x)
+    lineF(p, "$1 = $2;$n", [a, y.rdLoc])
 
 proc genMove(p: PProc; n: PNode; r: var TCompRes) =
   var a: TCompRes = default(TCompRes)
@@ -2234,7 +2278,7 @@ proc genMove(p: PProc; n: PNode; r: var TCompRes) =
   r.res = p.getTemp()
   gen(p, n[1], a)
   lineF(p, "$1 = $2;$n", [r.rdLoc, a.rdLoc])
-  genReset(p, n)
+  genWasMoved(p, n)
   #lineF(p, "$1 = $2;$n", [dest.rdLoc, src.rdLoc])
 
 proc genDup(p: PProc; n: PNode; r: var TCompRes) =
@@ -2407,6 +2451,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mMulSet: binaryExpr(p, n, r, "SetMul", "SetMul($1, $2)")
   of mPlusSet: binaryExpr(p, n, r, "SetPlus", "SetPlus($1, $2)")
   of mMinusSet: binaryExpr(p, n, r, "SetMinus", "SetMinus($1, $2)")
+  of mXorSet: binaryExpr(p, n, r, "SetXor", "SetXor($1, $2)")
   of mIncl: binaryExpr(p, n, r, "", "$1[$2] = true")
   of mExcl: binaryExpr(p, n, r, "", "delete $1[$2]")
   of mInSet:
@@ -2415,7 +2460,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mNewSeqOfCap: unaryExpr(p, n, r, "", "[]")
   of mOf: genOf(p, n, r)
   of mDefault, mZeroDefault: genDefault(p, n, r)
-  of mReset, mWasMoved: genReset(p, n)
+  of mWasMoved: genWasMoved(p, n)
   of mEcho: genEcho(p, n, r)
   of mNLen..mNError, mSlurp, mStaticExec:
     localError(p.config, n.info, errXMustBeCompileTime % n[0].sym.name.s)
@@ -2528,17 +2573,17 @@ proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) =
     let val = it[1]
     gen(p, val, a)
     var f = it[0].sym
-    if f.loc.r == "": f.loc.r = mangleName(p.module, f)
+    if f.loc.snippet == "": f.loc.snippet = mangleName(p.module, f)
     fieldIDs.incl(lookupFieldAgain(n.typ.skipTypes({tyDistinct}), f).id)
 
     let typ = val.typ.skipTypes(abstractInst)
     if a.typ == etyBaseIndex:
-      initList.addf("$#: [$#, $#]", [f.loc.r, a.address, a.res])
+      initList.addf("$#: [$#, $#]", [f.loc.snippet, a.address, a.res])
     else:
       if not needsNoCopy(p, val):
         useMagic(p, "nimCopy")
         a.res = "nimCopy(null, $1, $2)" % [a.rdLoc, genTypeInfo(p, typ)]
-      initList.addf("$#: $#", [f.loc.r, a.res])
+      initList.addf("$#: $#", [f.loc.snippet, a.res])
   let t = skipTypes(n.typ, abstractInst + skipPtrs)
   createObjInitList(p, t, fieldIDs, initList)
   r.res = ("{$1}") % [initList]
@@ -2597,7 +2642,13 @@ proc genRangeChck(p: PProc, n: PNode, r: var TCompRes, magic: string) =
   let src = skipTypes(n[0].typ, abstractVarRange)
   let dest = skipTypes(n.typ, abstractVarRange)
   if optRangeCheck notin p.options:
-    return
+    if optJsBigInt64 in p.config.globalOptions and
+          dest.kind in {tyUInt..tyUInt32, tyInt..tyInt32} and
+          src.kind in {tyInt64, tyUInt64}:
+      # conversions to Number are kept
+      r.res = "Number($1)" % [r.res]
+    else:
+      discard
   elif dest.kind in {tyUInt..tyUInt64} and checkUnsignedConversions notin p.config.legacyFeatures:
     if src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
       r.res = "BigInt.asUintN($1, $2)" % [$(dest.size * 8), r.res]
@@ -2692,6 +2743,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
   #if gVerbosity >= 3:
   #  echo "BEGIN generating code for: " & prc.name.s
   var p = newProc(oldProc.g, oldProc.module, prc.ast, prc.options)
+  p.up = oldProc
   var returnStmt: Rope = ""
   var resultAsgn: Rope = ""
   var name = mangleName(p.module, prc)
@@ -2777,9 +2829,9 @@ proc genPragma(p: PProc, n: PNode) =
     case whichPragma(it)
     of wEmit: genAsmOrEmitStmt(p, it[1])
     of wPush:
-      processPushBackendOption(p.optionsStack, p.options, n, i+1)
+      processPushBackendOption(p.config, p.optionsStack, p.options, n, i+1)
     of wPop:
-      processPopBackendOption(p.optionsStack, p.options)
+      processPopBackendOption(p.config, p.optionsStack, p.options)
     else: discard
 
 proc genCast(p: PProc, n: PNode, r: var TCompRes) =
@@ -2915,14 +2967,17 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     else:
       genCall(p, n, r)
   of nkClosure:
-    let tmp = getTemp(p)
-    var a: TCompRes = default(TCompRes)
-    var b: TCompRes = default(TCompRes)
-    gen(p, n[0], a)
-    gen(p, n[1], b)
-    lineF(p, "$1 = $2.bind($3); $1.ClP_0 = $2; $1.ClE_0 = $3;$n", [tmp, a.rdLoc, b.rdLoc])
-    r.res = tmp
-    r.kind = resVal
+    if jsNoLambdaLifting in p.config.legacyFeatures:
+      gen(p, n[0], r)
+    else:
+      let tmp = getTemp(p)
+      var a: TCompRes = default(TCompRes)
+      var b: TCompRes = default(TCompRes)
+      gen(p, n[0], a)
+      gen(p, n[1], b)
+      lineF(p, "$1 = $2.bind($3); $1.ClP_0 = $2; $1.ClE_0 = $3;$n", [tmp, a.rdLoc, b.rdLoc])
+      r.res = tmp
+      r.kind = resVal
   of nkCurly: genSetConstr(p, n, r)
   of nkBracket: genArrayConstr(p, n, r)
   of nkPar, nkTupleConstr: genTupleConstr(p, n, r)
@@ -2953,7 +3008,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkLambdaKinds:
     let s = n[namePos].sym
     discard mangleName(p.module, s)
-    r.res = s.loc.r
+    r.res = s.loc.snippet
     if lfNoDecl in s.loc.flags or s.magic notin generatedMagics: discard
     elif not p.g.generatedSyms.containsOrIncl(s.id):
       p.locals.add(genProc(p, s))
@@ -2993,11 +3048,8 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkRaiseStmt: genRaiseStmt(p, n)
   of nkTypeSection, nkCommentStmt, nkIncludeStmt,
      nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
-     nkFromStmt, nkTemplateDef, nkMacroDef, nkStaticStmt,
+     nkFromStmt, nkTemplateDef, nkMacroDef, nkIteratorDef, nkStaticStmt,
      nkMixinStmt, nkBindStmt: discard
-  of nkIteratorDef:
-    if n[0].sym.typ.callConv == TCallingConvention.ccClosure:
-      globalError(p.config, n.info, "Closure iterators are not supported by JS backend!")
   of nkPragma: genPragma(p, n)
   of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef:
     var s = n[namePos].sym
@@ -3005,7 +3057,9 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
       genSym(p, n[namePos], r)
       r.res = ""
   of nkGotoState, nkState:
-    globalError(p.config, n.info, "First class iterators not implemented")
+    globalError(p.config, n.info, "not implemented")
+  of nkBreakState:
+    genBreakState(p, n[0], r)
   of nkPragmaBlock: gen(p, n.lastSon, r)
   of nkComesFrom:
     discard "XXX to implement for better stack traces"
@@ -3112,15 +3166,6 @@ proc wholeCode(graph: ModuleGraph; m: BModule): Rope =
 
   result = globals.typeInfo & globals.constants & globals.code
 
-proc getClassName(t: PType): Rope =
-  var s = t.sym
-  if s.isNil or sfAnon in s.flags:
-    s = skipTypes(t, abstractPtrs).sym
-  if s.isNil or sfAnon in s.flags:
-    doAssert(false, "cannot retrieve class name")
-  if s.loc.r != "": result = s.loc.r
-  else: result = rope(s.name.s)
-
 proc finalJSCodeGen*(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   ## Finalize JS code generation of a Nim module.
   ## Param `n` may contain nodes returned from the last module close call.

compiler/lambdalifting.nim

@@ -145,11 +145,13 @@ proc createStateField(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym =
   result = newSym(skField, getIdent(g.cache, ":state"), idgen, iter, iter.info)
   result.typ = createClosureIterStateType(g, iter, idgen)
 
+template isIterator*(owner: PSym): bool =
+  owner.kind == skIterator and owner.typ.callConv == ccClosure
+
 proc createEnvObj(g: ModuleGraph; idgen: IdGenerator; owner: PSym; info: TLineInfo): PType =
-  # YYY meh, just add the state field for every closure for now, it's too
-  # hard to figure out if it comes from a closure iterator:
   result = createObj(g, idgen, owner, info, final=false)
-  rawAddField(result, createStateField(g, owner, idgen))
+  if owner.isIterator:
+    rawAddField(result, createStateField(g, owner, idgen))
 
 proc getClosureIterResult*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym =
   if resultPos < iter.ast.len:
@@ -172,26 +174,23 @@ proc addHiddenParam(routine: PSym, param: PSym) =
   assert sfFromGeneric in param.flags
   #echo "produced environment: ", param.id, " for ", routine.id
 
-proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
+proc getEnvParam*(routine: PSym): PSym =
+  if routine.ast.isNil: return nil
   let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName:
     result = hidden.sym
     assert sfFromGeneric in result.flags
   else:
+    result = nil
+
+proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
+  result = getEnvParam(routine)
+  if result.isNil:
     # writeStackTrace()
     localError(g.config, routine.info, "internal error: could not find env param for " & routine.name.s)
     result = routine
 
-proc getEnvParam*(routine: PSym): PSym =
-  let params = routine.ast[paramsPos]
-  let hidden = lastSon(params)
-  if hidden.kind == nkSym and hidden.sym.name.s == paramName:
-    result = hidden.sym
-    assert sfFromGeneric in result.flags
-  else:
-    result = nil
-
 proc interestingVar(s: PSym): bool {.inline.} =
   result = s.kind in {skVar, skLet, skTemp, skForVar, skParam, skResult} and
     sfGlobal notin s.flags and
@@ -230,14 +229,10 @@ proc makeClosure*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; env: PNode; inf
   if tfHasAsgn in result.typ.flags or optSeqDestructors in g.config.globalOptions:
     prc.flags.incl sfInjectDestructors
 
-proc interestingIterVar(s: PSym): bool {.inline.} =
-  # XXX optimization: Only lift the variable if it lives across
-  # yield/return boundaries! This can potentially speed up
-  # closure iterators quite a bit.
-  result = s.kind in {skResult, skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags
-
-template isIterator*(owner: PSym): bool =
-  owner.kind == skIterator and owner.typ.callConv == ccClosure
+template liftingHarmful(conf: ConfigRef; owner: PSym): bool =
+  ## lambda lifting can be harmful for JS-like code generators.
+  let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
+  jsNoLambdaLifting in conf.legacyFeatures and conf.backend == backendJs and not isCompileTime
 
 proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; idgen: IdGenerator; owner: PSym) =
   if owner.kind != skMacro:
@@ -255,11 +250,11 @@ proc genCreateEnv(env: PNode): PNode =
 
 proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
+  if liftingHarmful(g.config, owner): return n
   let iter = n.sym
   assert iter.isIterator
 
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-
+  result = newNodeIT(nkStmtListExpr, n.info, iter.typ)
   let hp = getHiddenParam(g, iter)
   var env: PNode
   if owner.isIterator:
@@ -280,15 +275,6 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN
   createTypeBoundOpsLL(g, env.typ, n.info, idgen, owner)
   result.add makeClosure(g, idgen, iter, env, n.info)
 
-proc freshVarForClosureIter*(g: ModuleGraph; s: PSym; idgen: IdGenerator; owner: PSym): PNode =
-  let envParam = getHiddenParam(g, owner)
-  let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr})
-  let field = addField(obj, s, g.cache, idgen)
-
-  var access = newSymNode(envParam)
-  assert obj.kind == tyObject
-  result = rawIndirectAccess(access, field, s.info)
-
 # ------------------ new stuff -------------------------------------------
 
 proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
@@ -298,8 +284,8 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
     localError(g.config, n.info,
       ("'$1' is of type <$2> which cannot be captured as it would violate memory" &
        " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases." &
-       " Consider using a <ref $2> which can be captured.") %
-      [s.name.s, typeToString(s.typ), g.config$s.info])
+       " Consider using a <ref T> which can be captured.") %
+      [s.name.s, typeToString(s.typ.skipTypes({tyVar})), g.config$s.info])
   elif not (owner.typ.isClosure or owner.isNimcall and not owner.isExplicitCallConv or isEnv):
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
       [s.name.s, owner.name.s, $owner.typ.callConv])
@@ -336,9 +322,13 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
                         info: TLineInfo): PType =
   result = c.ownerToType.getOrDefault(owner.id)
   if result.isNil:
-    result = newType(tyRef, c.idgen, owner)
-    let obj = createEnvObj(c.graph, c.idgen, owner, info)
-    rawAddSon(result, obj)
+    let env = getEnvParam(owner)
+    if env.isNil or not owner.isIterator:
+      result = newType(tyRef, c.idgen, owner)
+      let obj = createEnvObj(c.graph, c.idgen, owner, info)
+      rawAddSon(result, obj)
+    else:
+      result = env.typ
     c.ownerToType[owner.id] = result
 
 proc asOwnedRef(c: var DetectionPass; t: PType): PType =
@@ -430,6 +420,12 @@ proc addClosureParam(c: var DetectionPass; fn: PSym; info: TLineInfo) =
     localError(c.graph.config, fn.info, "internal error: inconsistent environment type")
   #echo "adding closure to ", fn.name.s
 
+proc iterEnvHasUpField(g: ModuleGraph, iter: PSym): bool =
+  let cp = getEnvParam(iter)
+  doAssert(cp != nil, "Env param not present in iter")
+  let upField = lookupInRecord(cp.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(g.cache, upName))
+  upField != nil
+
 proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
   case n.kind
   of nkSym:
@@ -448,22 +444,12 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
         let body = transformBody(c.graph, c.idgen, s, {useCache})
         detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
-    let innerClosure = innerProc and s.typ.callConv == ccClosure and not s.isIterator
+    let innerClosure = innerProc and s.typ.callConv == ccClosure and (not s.isIterator or iterEnvHasUpField(c.graph, s))
     let interested = interestingVar(s)
     if ow == owner:
       if owner.isIterator:
         c.somethingToDo = true
         addClosureParam(c, owner, n.info)
-        if interestingIterVar(s):
-          if not c.capturedVars.contains(s.id):
-            if not c.inTypeOf: c.capturedVars.incl(s.id)
-            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
-            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef, tyPtr})
-
-            if s.name.id == getIdent(c.graph.cache, ":state").id:
-              obj.n[0].sym.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
-            else:
-              discard addField(obj, s, c.graph.cache, c.idgen)
     # direct or indirect dependency:
     elif innerClosure or interested:
       discard """
@@ -623,7 +609,7 @@ proc rawClosureCreation(owner: PSym;
       let unowned = c.unownedEnvVars[owner.id]
       assert unowned != nil
       let env2 = copyTree(env)
-      env2.typ = unowned.typ
+      env2.typ() = unowned.typ
       result.add newAsgnStmt(unowned, env2, env.info)
       createTypeBoundOpsLL(d.graph, unowned.typ, env.info, d.idgen, owner)
 
@@ -663,16 +649,27 @@ proc finishClosureCreation(owner: PSym; d: var DetectionPass; c: LiftingPass;
     res.add newAsgnStmt(unowned, nilLit, info)
     createTypeBoundOpsLL(d.graph, unowned.typ, info, d.idgen, owner)
 
-proc closureCreationForIter(iter: PNode;
+proc getUpForIter(g: ModuleGraph; owner, iterOwner: PSym, expectedUpTyp: PType): PNode =
+  var p = getHiddenParam(g, owner)
+  var res = p.newSymNode
+  while res.typ.skipTypes({tyOwned, tyRef, tyPtr}) != expectedUpTyp:
+    let upField = lookupInRecord(p.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(g.cache, upName))
+    if upField == nil:
+      return nil
+    p = upField
+    res = rawIndirectAccess(res, upField, p.info)
+  res
+
+proc closureCreationForIter(owner: PSym, iter: PNode;
                             d: var DetectionPass; c: var LiftingPass): PNode =
   result = newNodeIT(nkStmtListExpr, iter.info, iter.sym.typ)
-  let owner = iter.sym.skipGenericOwner
-  var v = newSym(skVar, getIdent(d.graph.cache, envName), d.idgen, owner, iter.info)
+  let iterOwner = iter.sym.skipGenericOwner
+  var v = newSym(skVar, getIdent(d.graph.cache, envName), d.idgen, iterOwner, iter.info)
   incl(v.flags, sfShadowed)
   v.typ = asOwnedRef(d, getHiddenParam(d.graph, iter.sym).typ)
   var vnode: PNode
-  if owner.isIterator:
-    let it = getHiddenParam(d.graph, owner)
+  if iterOwner.isIterator:
+    let it = getHiddenParam(d.graph, iterOwner)
     addUniqueField(it.typ.skipTypes({tyOwned, tyRef, tyPtr}), v, d.graph.cache, d.idgen)
     vnode = indirectAccess(newSymNode(it), v, v.info)
   else:
@@ -681,12 +678,14 @@ proc closureCreationForIter(iter: PNode;
     addVar(vs, vnode)
     result.add(vs)
   result.add genCreateEnv(vnode)
-  createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, d.idgen, owner)
+  createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, d.idgen, iterOwner)
 
   let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
-    let u = setupEnvVar(owner, d, c, iter.info)
-    if u.typ.skipTypes({tyOwned, tyRef, tyPtr}) == upField.typ.skipTypes({tyOwned, tyRef, tyPtr}):
+    let expectedUpTyp = upField.typ.skipTypes({tyOwned, tyRef, tyPtr})
+    let u = if iterOwner == owner: setupEnvVar(iterOwner, d, c, iter.info)
+            else: getUpForIter(d.graph, owner, iterOwner, expectedUpTyp)
+    if u != nil and u.typ.skipTypes({tyOwned, tyRef, tyPtr}) == expectedUpTyp:
       result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
                  u, iter.info))
     else:
@@ -720,7 +719,7 @@ proc symToClosure(n: PNode; owner: PSym; d: var DetectionPass;
     let available = getHiddenParam(d.graph, owner)
     result = makeClosure(d.graph, d.idgen, s, available.newSymNode, n.info)
   elif s.isIterator:
-    result = closureCreationForIter(n, d, c)
+    result = closureCreationForIter(owner, n, d, c)
   elif s.skipGenericOwner == owner:
     # direct dependency, so use the outer's env variable:
     result = makeClosure(d.graph, d.idgen, s, setupEnvVar(owner, d, c, n.info), n.info)
@@ -767,8 +766,6 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
     elif s.id in d.capturedVars:
       if s.owner != owner:
         result = accessViaEnvParam(d.graph, n, owner)
-      elif owner.isIterator and interestingIterVar(s):
-        result = accessViaEnvParam(d.graph, n, owner)
       else:
         result = accessViaEnvVar(n, owner, d, c)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
@@ -789,7 +786,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
       let oldInContainer = c.inContainer
       c.inContainer = 0
       let m = newSymNode(n[namePos].sym)
-      m.typ = n.typ
+      m.typ() = n.typ
       result = liftCapturedVars(m, owner, d, c)
       c.inContainer = oldInContainer
   of nkHiddenStdConv:
@@ -879,12 +876,16 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
                   idgen: IdGenerator; flags: TransformFlags): PNode =
   let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro
 
-  if body.kind == nkEmpty or
+  if body.kind == nkEmpty or (jsNoLambdaLifting in g.config.legacyFeatures and
+      g.config.backend == backendJs and not isCompileTime) or
       (fn.skipGenericOwner.kind != skModule and force notin flags):
 
     # ignore forward declaration:
     result = body
     tooEarly = true
+    if fn.isIterator:
+      var d = initDetectionPass(g, fn, idgen)
+      addClosureParam(d, fn, body.info)
   else:
     var d = initDetectionPass(g, fn, idgen)
     detectCapturedVars(body, fn, d)
@@ -939,6 +940,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
           break
         ...
     """
+  if liftingHarmful(g.config, owner): return body
   if not (body.kind == nkForStmt and body[^2].kind in nkCallKinds):
     localError(g.config, body.info, "ignored invalid for loop")
     return body

compiler/layeredtable.nim

@@ -0,0 +1,82 @@
+import std/tables
+import ast
+
+type
+  LayeredIdTableObj* {.acyclic.} = object
+    ## stack of type binding contexts implemented as a linked list
+    topLayer*: TypeMapping
+      ## the mappings on the current layer
+    nextLayer*: ref LayeredIdTableObj
+      ## the parent type binding context, possibly `nil`
+    previousLen*: int
+      ## total length of the bindings up to the parent layer,
+      ## used to track if new bindings were added
+
+const useRef = not defined(gcDestructors)
+  # implementation detail, only arc/orc doesn't cause issues when
+  # using LayeredIdTable as an object and not a ref
+
+when useRef:
+  type LayeredIdTable* = ref LayeredIdTableObj
+else:
+  type LayeredIdTable* = LayeredIdTableObj
+
+proc initLayeredTypeMap*(pt: sink TypeMapping = initTypeMapping()): LayeredIdTable =
+  result = LayeredIdTable(topLayer: pt, nextLayer: nil)
+
+proc shallowCopy*(pt: LayeredIdTable): LayeredIdTable {.inline.} =
+  ## copies only the type bindings of the current layer, but not any parent layers,
+  ## useful for write-only bindings
+  result = LayeredIdTable(topLayer: pt.topLayer, nextLayer: pt.nextLayer, previousLen: pt.previousLen)
+
+proc currentLen*(pt: LayeredIdTable): int =
+  ## the sum of the cached total binding count of the parents and
+  ## the current binding count, just used to track if bindings were added
+  pt.previousLen + pt.topLayer.len
+
+proc newTypeMapLayer*(pt: LayeredIdTable): LayeredIdTable =
+  result = LayeredIdTable(topLayer: initTable[ItemId, PType](), previousLen: pt.currentLen)
+  when useRef:
+    result.nextLayer = pt
+  else:
+    new(result.nextLayer)
+    result.nextLayer[] = pt
+
+proc setToPreviousLayer*(pt: var LayeredIdTable) {.inline.} =
+  when useRef:
+    pt = pt.nextLayer
+  else:
+    when defined(gcDestructors):
+      pt = pt.nextLayer[]
+    else:
+      # workaround refc
+      let tmp = pt.nextLayer[]
+      pt = tmp
+
+proc lookup(typeMap: ref LayeredIdTableObj, key: ItemId): PType =
+  result = nil
+  var tm = typeMap
+  while tm != nil:
+    result = getOrDefault(tm.topLayer, key)
+    if result != nil: return
+    tm = tm.nextLayer
+
+template lookup*(typeMap: ref LayeredIdTableObj, key: PType): PType =
+  ## recursively looks up binding of `key` in all parent layers
+  lookup(typeMap, key.itemId)
+
+when not useRef:
+  proc lookup(typeMap: LayeredIdTableObj, key: ItemId): PType {.inline.} =
+    result = getOrDefault(typeMap.topLayer, key)
+    if result == nil and typeMap.nextLayer != nil:
+      result = lookup(typeMap.nextLayer, key)
+
+  template lookup*(typeMap: LayeredIdTableObj, key: PType): PType =
+    lookup(typeMap, key.itemId)
+
+proc put(typeMap: var LayeredIdTable, key: ItemId, value: PType) {.inline.} =
+  typeMap.topLayer[key] = value
+
+template put*(typeMap: var LayeredIdTable, key, value: PType) =
+  ## binds `key` to `value` only in current layer
+  put(typeMap, key.itemId, value)

compiler/liftdestructors.nim

@@ -40,7 +40,7 @@ template asink*(t: PType): PSym = getAttachedOp(c.g, t, attachedSink)
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
 proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
-              info: TLineInfo; idgen: IdGenerator): PSym
+              info: TLineInfo; idgen: IdGenerator; isDistinct = false): PSym
 
 proc createTypeBoundOps*(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo;
                          idgen: IdGenerator)
@@ -49,7 +49,7 @@ proc at(a, i: PNode, elemType: PType): PNode =
   result = newNodeI(nkBracketExpr, a.info, 2)
   result[0] = a
   result[1] = i
-  result.typ = elemType
+  result.typ() = elemType
 
 proc destructorOverridden(g: ModuleGraph; t: PType): bool =
   let op = getAttachedOp(g, t, attachedDestructor)
@@ -68,7 +68,7 @@ proc dotField(x: PNode, f: PSym): PNode =
   else:
     result[0] = x
   result[1] = newSymNode(f, x.info)
-  result.typ = f.typ
+  result.typ() = f.typ
 
 proc newAsgnStmt(le, ri: PNode): PNode =
   result = newNodeI(nkAsgn, le.info, 2)
@@ -88,7 +88,7 @@ proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add newAsgnStmt(x, y)
   elif c.kind == attachedDestructor and c.addMemReset:
     let call = genBuiltin(c, mDefault, "default", x)
-    call.typ = t
+    call.typ() = t
     body.add newAsgnStmt(x, call)
   elif c.kind == attachedWasMoved:
     body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
@@ -105,7 +105,7 @@ proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
   result = newNodeI(nkWhileStmt, c.info, 2)
   let cmp = genBuiltin(c, mLtI, "<", i)
   cmp.add genLen(c.g, dest)
-  cmp.typ = getSysType(c.g, c.info, tyBool)
+  cmp.typ() = getSysType(c.g, c.info, tyBool)
   result[0] = cmp
   result[1] = newNodeI(nkStmtList, c.info)
 
@@ -127,10 +127,10 @@ proc genContainerOf(c: var TLiftCtx; objType: PType, field, x: PSym): PNode =
   dotExpr.add newSymNode(field)
 
   let offsetOf = genBuiltin(c, mOffsetOf, "offsetof", dotExpr)
-  offsetOf.typ = intType
+  offsetOf.typ() = intType
 
   let minusExpr = genBuiltin(c, mSubI, "-", castExpr1)
-  minusExpr.typ = intType
+  minusExpr.typ() = intType
   minusExpr.add offsetOf
 
   let objPtr = makePtrType(objType.owner, objType, c.idgen)
@@ -157,7 +157,7 @@ proc genWasMovedCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
   result.add(newSymNode(op))
   result.add genAddr(c, x)
 
-proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
+proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool, enforceWasMoved = false) =
   case n.kind
   of nkSym:
     if c.filterDiscriminator != nil: return
@@ -167,6 +167,8 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
         enforceDefaultOp:
       defaultOp(c, f.typ, body, x.dotField(f), b)
     else:
+      if enforceWasMoved:
+        body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x.dotField(f))
       fillBody(c, f.typ, body, x.dotField(f), b)
   of nkNilLit: discard
   of nkRecCase:
@@ -205,7 +207,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       branch[^1] = newNodeI(nkStmtList, c.info)
 
       fillBodyObj(c, n[i].lastSon, branch[^1], x, y,
-                  enforceDefaultOp = localEnforceDefaultOp)
+                  enforceDefaultOp = localEnforceDefaultOp, enforceWasMoved = c.kind == attachedAsgn)
       if branch[^1].len == 0: inc emptyBranches
       caseStmt.add(branch)
     if emptyBranches != n.len-1:
@@ -216,13 +218,22 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       fillBodyObj(c, n[0], body, x, y, enforceDefaultOp = false)
     c.filterDiscriminator = oldfilterDiscriminator
   of nkRecList:
-    for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp)
+    for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp, enforceWasMoved)
   else:
     illFormedAstLocal(n, c.g.config)
 
 proc fillBodyObjTImpl(c: var TLiftCtx; t: PType, body, x, y: PNode) =
   if t.baseClass != nil:
-    fillBody(c, skipTypes(t.baseClass, abstractPtrs), body, x, y)
+    let dest = newNodeIT(nkHiddenSubConv, c.info, t.baseClass)
+    dest.add newNodeI(nkEmpty, c.info)
+    dest.add x
+    var src = y
+    if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink}:
+      src = newNodeIT(nkHiddenSubConv, c.info, t.baseClass)
+      src.add newNodeI(nkEmpty, c.info)
+      src.add y
+
+    fillBody(c, skipTypes(t.baseClass, abstractPtrs), body, dest, src)
   fillBodyObj(c, t.n, body, x, y, enforceDefaultOp = false)
 
 proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
@@ -254,7 +265,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
       # because the wasMoved(dest) call would zero out src, if dest aliases src.
       var cond = newTree(nkCall, newSymNode(c.g.getSysMagic(c.info, "==", mEqRef)),
         newTreeIT(nkAddr, c.info, makePtrType(c.fn, x.typ, c.idgen), x), newTreeIT(nkAddr, c.info, makePtrType(c.fn, y.typ, c.idgen), y))
-      cond.typ = getSysType(c.g, x.info, tyBool)
+      cond.typ() = getSysType(c.g, x.info, tyBool)
       body.add genIf(c, cond, newTreeI(nkReturnStmt, c.info, newNodeI(nkEmpty, c.info)))
     var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
     temp.typ = x.typ
@@ -279,12 +290,13 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     c.kind = attachedDestructor
     fillBodyObjTImpl(c, t, body, blob, y)
     c.kind = prevKind
+
   else:
     fillBodyObjTImpl(c, t, body, x, y)
 
 proc boolLit*(g: ModuleGraph; info: TLineInfo; value: bool): PNode =
   result = newIntLit(g, info, ord value)
-  result.typ = getSysType(g, info, tyBool)
+  result.typ() = getSysType(g, info, tyBool)
 
 proc getCycleParam(c: TLiftCtx): PNode =
   assert c.kind in {attachedAsgn, attachedDup}
@@ -533,18 +545,18 @@ proc newSeqCall(c: var TLiftCtx; x, y: PNode): PNode =
   # don't call genAddr(c, x) here:
   result = genBuiltin(c, mNewSeq, "newSeq", x)
   let lenCall = genBuiltin(c, mLengthSeq, "len", y)
-  lenCall.typ = getSysType(c.g, x.info, tyInt)
+  lenCall.typ() = getSysType(c.g, x.info, tyInt)
   result.add lenCall
 
 proc setLenStrCall(c: var TLiftCtx; x, y: PNode): PNode =
   let lenCall = genBuiltin(c, mLengthStr, "len", y)
-  lenCall.typ = getSysType(c.g, x.info, tyInt)
+  lenCall.typ() = getSysType(c.g, x.info, tyInt)
   result = genBuiltin(c, mSetLengthStr, "setLen", x) # genAddr(g, x))
   result.add lenCall
 
 proc setLenSeqCall(c: var TLiftCtx; t: PType; x, y: PNode): PNode =
   let lenCall = genBuiltin(c, mLengthSeq, "len", y)
-  lenCall.typ = getSysType(c.g, x.info, tyInt)
+  lenCall.typ() = getSysType(c.g, x.info, tyInt)
   var op = getSysMagic(c.g, x.info, "setLen", mSetLengthSeq)
   op = instantiateGeneric(c, op, t, t)
   result = newTree(nkCall, newSymNode(op, x.info), x, lenCall)
@@ -567,7 +579,7 @@ proc checkSelfAssignment(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       newTreeIT(nkAddr, c.info, makePtrType(c.fn, x.typ, c.idgen), x),
       newTreeIT(nkAddr, c.info, makePtrType(c.fn, y.typ, c.idgen), y)
       )
-  cond.typ = getSysType(c.g, c.info, tyBool)
+  cond.typ() = getSysType(c.g, c.info, tyBool)
   body.add genIf(c, cond, newTreeI(nkReturnStmt, c.info, newNodeI(nkEmpty, c.info)))
 
 proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
@@ -708,7 +720,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if isFinal(elemType):
     addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr))
     var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
-    alignOf.typ = getSysType(c.g, c.info, tyInt)
+    alignOf.typ() = getSysType(c.g, c.info, tyInt)
     actions.add callCodegenProc(c.g, "nimRawDispose", c.info, tmp, alignOf)
   else:
     addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(tmp, nkDerefExpr))
@@ -718,7 +730,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if isCyclic:
     if isFinal(elemType):
       let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
-      typInfo.typ = getSysType(c.g, c.info, tyPointer)
+      typInfo.typ() = getSysType(c.g, c.info, tyPointer)
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
     else:
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, tmp)
@@ -726,7 +738,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     cond = callCodegenProc(c.g, "nimDecRefIsLastDyn", c.info, x)
   else:
     cond = callCodegenProc(c.g, "nimDecRefIsLast", c.info, x)
-  cond.typ = getSysType(c.g, x.info, tyBool)
+  cond.typ() = getSysType(c.g, x.info, tyBool)
 
   case c.kind
   of attachedSink:
@@ -753,7 +765,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     if isCyclic:
       if isFinal(elemType):
         let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
-        typInfo.typ = getSysType(c.g, c.info, tyPointer)
+        typInfo.typ() = getSysType(c.g, c.info, tyPointer)
         body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y)
       else:
         # If the ref is polymorphic we have to account for this
@@ -774,7 +786,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   ## Closures are really like refs except they always use a virtual destructor
   ## and we need to do the refcounting only on the ref field which we call 'xenv':
   let xenv = genBuiltin(c, mAccessEnv, "accessEnv", x)
-  xenv.typ = getSysType(c.g, c.info, tyPointer)
+  xenv.typ() = getSysType(c.g, c.info, tyPointer)
 
   let isCyclic = c.g.config.selectedGC == gcOrc
   let tmp =
@@ -790,7 +802,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     if isCyclic: "nimDecRefIsLastCyclicDyn"
     else: "nimDecRefIsLast"
   let cond = callCodegenProc(c.g, decRefProc, c.info, tmp)
-  cond.typ = getSysType(c.g, x.info, tyBool)
+  cond.typ() = getSysType(c.g, x.info, tyBool)
 
   case c.kind
   of attachedSink:
@@ -802,7 +814,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add newAsgnStmt(x, y)
   of attachedAsgn:
     let yenv = genBuiltin(c, mAccessEnv, "accessEnv", y)
-    yenv.typ = getSysType(c.g, c.info, tyPointer)
+    yenv.typ() = getSysType(c.g, c.info, tyPointer)
     if isCyclic:
       body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRefCyclic", c.info, yenv, getCycleParam(c)))
       body.add newAsgnStmt(x, y)
@@ -814,7 +826,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add newAsgnStmt(x, y)
   of attachedDup:
     let yenv = genBuiltin(c, mAccessEnv, "accessEnv", y)
-    yenv.typ = getSysType(c.g, c.info, tyPointer)
+    yenv.typ() = getSysType(c.g, c.info, tyPointer)
     if isCyclic:
       body.add newAsgnStmt(x, y)
       body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRefCyclic", c.info, yenv, getCycleParam(c)))
@@ -866,7 +878,7 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if isFinal(elemType):
     addDestructorCall(c, elemType, actions, genDeref(x, nkDerefExpr))
     var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
-    alignOf.typ = getSysType(c.g, c.info, tyInt)
+    alignOf.typ() = getSysType(c.g, c.info, tyInt)
     actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x, alignOf)
   else:
     addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(x, nkDerefExpr))
@@ -889,14 +901,14 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     # a big problem is that we don't know the environment's type here, so we
     # have to go through some indirection; we delegate this to the codegen:
     let call = newNodeI(nkCall, c.info, 2)
-    call.typ = t
+    call.typ() = t
     call[0] = newSymNode(createMagic(c.g, c.idgen, "deepCopy", mDeepCopy))
     call[1] = y
     body.add newAsgnStmt(x, call)
   elif (optOwnedRefs in c.g.config.globalOptions and
       optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}:
     let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
-    xx.typ = getSysType(c.g, c.info, tyPointer)
+    xx.typ() = getSysType(c.g, c.info, tyPointer)
     case c.kind
     of attachedSink:
       # we 'nil' y out afterwards so we *need* to take over its reference
@@ -905,13 +917,13 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add newAsgnStmt(x, y)
     of attachedAsgn:
       let yy = genBuiltin(c, mAccessEnv, "accessEnv", y)
-      yy.typ = getSysType(c.g, c.info, tyPointer)
+      yy.typ() = getSysType(c.g, c.info, tyPointer)
       body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy))
       body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
       body.add newAsgnStmt(x, y)
     of attachedDup:
       let yy = genBuiltin(c, mAccessEnv, "accessEnv", y)
-      yy.typ = getSysType(c.g, c.info, tyPointer)
+      yy.typ() = getSysType(c.g, c.info, tyPointer)
       body.add newAsgnStmt(x, y)
       body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy))
     of attachedDestructor:
@@ -926,7 +938,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
 proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
-  xx.typ = getSysType(c.g, c.info, tyPointer)
+  xx.typ() = getSysType(c.g, c.info, tyPointer)
   var actions = newNodeI(nkStmtList, c.info)
   #discard addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(xx))
   actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, xx)
@@ -1034,7 +1046,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     else:
       discard "cannot copy openArray"
 
-  of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
+  of tyFromExpr, tyError, tyBuiltInTypeClass, tyUserTypeClass,
      tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything,
      tyGenericParam, tyGenericBody, tyNil, tyUntyped, tyTyped,
      tyTypeDesc, tyGenericInvocation, tyForward, tyStatic:
@@ -1051,7 +1063,9 @@ proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
   assert typ.kind == tyDistinct
   let baseType = typ.elementType
   if getAttachedOp(g, baseType, kind) == nil:
-    discard produceSym(g, c, baseType, kind, info, idgen)
+    # TODO: fixme `isDistinct` is a fix for #23552; remove it after
+    # `-d:nimPreviewNonVarDestructor` becomes the default
+    discard produceSym(g, c, baseType, kind, info, idgen, isDistinct = true)
   result = getAttachedOp(g, baseType, kind)
   setAttachedOp(g, idgen.module, typ, kind, result)
 
@@ -1090,7 +1104,7 @@ proc symDupPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttache
   incl result.flags, sfGeneratedOp
 
 proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
-              info: TLineInfo; idgen: IdGenerator; isDiscriminant = false): PSym =
+              info: TLineInfo; idgen: IdGenerator; isDiscriminant = false; isDistinct = false): PSym =
   if kind == attachedDup:
     return symDupPrototype(g, typ, owner, kind, info, idgen)
 
@@ -1101,7 +1115,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
                    idgen, result, info)
 
   if kind == attachedDestructor and g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc} and
-     ((g.config.isDefined("nimPreviewNonVarDestructor") and not isDiscriminant) or typ.kind in {tyRef, tyString, tySequence}):
+     ((g.config.isDefined("nimPreviewNonVarDestructor") and not isDiscriminant) or (typ.kind in {tyRef, tyString, tySequence} and not isDistinct)):
     dest.typ = typ
   else:
     dest.typ = makeVarType(typ.owner, typ, idgen)
@@ -1138,18 +1152,18 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
 proc genTypeFieldCopy(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let xx = genBuiltin(c, mAccessTypeField, "accessTypeField", x)
   let yy = genBuiltin(c, mAccessTypeField, "accessTypeField", y)
-  xx.typ = getSysType(c.g, c.info, tyPointer)
-  yy.typ = xx.typ
+  xx.typ() = getSysType(c.g, c.info, tyPointer)
+  yy.typ() = xx.typ
   body.add newAsgnStmt(xx, yy)
 
 proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
-              info: TLineInfo; idgen: IdGenerator): PSym =
+              info: TLineInfo; idgen: IdGenerator; isDistinct = false): PSym =
   if typ.kind == tyDistinct:
     return produceSymDistinctType(g, c, typ, kind, info, idgen)
 
   result = getAttachedOp(g, typ, kind)
   if result == nil:
-    result = symPrototype(g, typ, typ.owner, kind, info, idgen)
+    result = symPrototype(g, typ, typ.owner, kind, info, idgen, isDistinct = isDistinct)
 
   var a = TLiftCtx(info: info, g: g, kind: kind, c: c, asgnForType: typ, idgen: idgen,
                    fn: result)

compiler/liftlocals.nim

@@ -32,12 +32,12 @@ proc interestingVar(s: PSym): bool {.inline.} =
 proc lookupOrAdd(c: var Ctx; s: PSym; info: TLineInfo): PNode =
   let field = addUniqueField(c.objType, s, c.cache, c.idgen)
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = c.objType
+  deref.typ() = c.objType
   deref.add(newSymNode(c.partialParam, info))
   result = newNodeI(nkDotExpr, info)
   result.add(deref)
   result.add(newSymNode(field))
-  result.typ = field.typ
+  result.typ() = field.typ
 
 proc liftLocals(n: PNode; i: int; c: var Ctx) =
   let it = n[i]

compiler/lineinfos.nim

@@ -92,9 +92,10 @@ type
     warnStmtListLambda = "StmtListLambda",
     warnBareExcept = "BareExcept",
     warnImplicitDefaultValue = "ImplicitDefaultValue",
-    warnGenericsIgnoredInjection = "GenericsIgnoredInjection",
+    warnIgnoredSymbolInjection = "IgnoredSymbolInjection",
     warnStdPrefix = "StdPrefix"
     warnUser = "User",
+    warnGlobalVarConstructorTemporary = "GlobalVarConstructorTemporary",
     # hints
     hintSuccess = "Success", hintSuccessX = "SuccessX",
     hintCC = "CC",
@@ -197,9 +198,10 @@ const
     warnStmtListLambda: "statement list expression assumed to be anonymous proc; this is deprecated, use `do (): ...` or `proc () = ...` instead",
     warnBareExcept: "$1",
     warnImplicitDefaultValue: "$1",
-    warnGenericsIgnoredInjection: "$1",
+    warnIgnoredSymbolInjection: "$1",
     warnStdPrefix: "$1 needs the 'std' prefix",
     warnUser: "$1",
+    warnGlobalVarConstructorTemporary: "global variable '$1' initialization requires a temporary variable",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
     hintSuccessX: "$build\n$loc lines; ${sec}s; $mem; proj: $project; out: $output",
@@ -266,6 +268,7 @@ const
   NotesVerbosity* = computeNotesVerbosity()
   errXMustBeCompileTime* = "'$1' can only be used in compile-time context"
   errArgsNeedRunOption* = "arguments can only be given if the '--run' option is selected"
+  errFloatToString* = "cannot convert '$1' to '$2'"
 
 type
   TFileInfo* = object

compiler/linter.nim

@@ -12,7 +12,7 @@
 import std/strutils
 from std/sugar import dup
 
-import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs, semdata, packages
+import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs, semdata, packages, modulegraphs
 export packages
 
 const
@@ -97,7 +97,7 @@ template styleCheckDef*(ctx: PContext; info: TLineInfo; sym: PSym; k: TSymKind)
   if optStyleCheck in ctx.config.options and # ignore if styleChecks are off
      {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # check only if hint/error is enabled
      hintName in ctx.config.notes and # ignore if name checks are not requested
-     ctx.config.belongsToProjectPackage(sym) and # ignore foreign packages
+     ctx.config.belongsToProjectPackageMaybeNil(getModule(ctx.graph, info.fileIndex)) and # ignore foreign packages
      optStyleUsages notin ctx.config.globalOptions and # ignore if requested to only check name usage
      sym.kind != skResult and # ignore `result`
      sym.kind != skTemp and # ignore temporary variables created by the compiler
@@ -138,7 +138,7 @@ template styleCheckUse*(ctx: PContext; info: TLineInfo; sym: PSym) =
   ## Check symbol uses match their definition's style.
   if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
      hintName in ctx.config.notes and # ignore if name checks are not requested
-     ctx.config.belongsToProjectPackage(sym) and # ignore foreign packages
+     ctx.config.belongsToProjectPackageMaybeNil(getModule(ctx.graph, info.fileIndex)) and # ignore foreign packages
      sym.kind != skTemp and # ignore temporary variables created by the compiler
      sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
      sfAnon notin sym.flags: # ignore temporary variables created by the compiler
@@ -154,5 +154,5 @@ template checkPragmaUse*(ctx: PContext; info: TLineInfo; w: TSpecialWord; pragma
   ## Note: This only applies to builtin pragmas, not user pragmas.
   if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
      hintName in ctx.config.notes and # ignore if name checks are not requested
-     (sym != nil and ctx.config.belongsToProjectPackage(sym)): # ignore foreign packages
+     ctx.config.belongsToProjectPackageMaybeNil(getModule(ctx.graph, info.fileIndex)): # ignore foreign packages
     checkPragmaUseImpl(ctx.config, info, w, pragmaName)

compiler/llstream.nim

@@ -68,6 +68,7 @@ when not declared(readLineFromStdin):
   # fallback implementation:
   proc readLineFromStdin(prompt: string, line: var string): bool =
     stdout.write(prompt)
+    stdout.flushFile()
     result = readLine(stdin, line)
     if not result:
       stdout.write("\n")

compiler/lookups.nim

@@ -50,7 +50,7 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
-        of nkSymChoices:
+        of nkSymChoices, nkOpenSym:
           if x[0].kind == nkSym:
             id.add(x[0].sym.name.s)
           else:
@@ -63,6 +63,8 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
       result = n[0].sym.name
     else:
       handleError(n, origin)
+  of nkOpenSym:
+    result = considerQuotedIdent(c, n[0], origin)
   else:
     handleError(n, origin)
 
@@ -561,23 +563,33 @@ proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) =
   var amb: bool = false
   discard errorUseQualifier(c, info, s, amb)
 
-proc errorUseQualifier*(c: PContext; info: TLineInfo; candidates: seq[PSym]; prefix = "use one of") =
-  var err = "ambiguous identifier: '" & candidates[0].name.s & "'"
+proc ambiguousIdentifierMsg*(candidates: seq[PSym], prefix = "use one of", indent = 0): string =
+  result = ""
+  for i in 0 ..< indent:
+    result.add(' ')
+  result.add "ambiguous identifier: '" & candidates[0].name.s & "'"
   var i = 0
   for candidate in candidates:
-    if i == 0: err.add " -- $1 the following:\n" % prefix
-    else: err.add "\n"
-    err.add "  " & candidate.owner.name.s & "." & candidate.name.s
-    err.add ": " & typeToString(candidate.typ)
+    if i == 0: result.add " -- $1 the following:\n" % prefix
+    else: result.add "\n"
+    for i in 0 ..< indent:
+      result.add(' ')
+    result.add "  " & candidate.owner.name.s & "." & candidate.name.s
+    result.add ": " & typeToString(candidate.typ)
     inc i
-  localError(c.config, info, errGenerated, err)
 
-proc errorUseQualifier*(c: PContext; info:TLineInfo; choices: PNode) =
+proc errorUseQualifier*(c: PContext; info: TLineInfo; candidates: seq[PSym]) =
+  localError(c.config, info, errGenerated, ambiguousIdentifierMsg(candidates))
+
+proc ambiguousIdentifierMsg*(choices: PNode, indent = 0): string =
   var candidates = newSeq[PSym](choices.len)
   let prefix = if choices[0].typ.kind != tyProc: "use one of" else: "you need a helper proc to disambiguate"
   for i, n in choices:
     candidates[i] = n.sym
-  errorUseQualifier(c, info, candidates, prefix)
+  result = ambiguousIdentifierMsg(candidates, prefix, indent)
+
+proc errorUseQualifier*(c: PContext; info:TLineInfo; choices: PNode) =
+  localError(c.config, info, errGenerated, ambiguousIdentifierMsg(choices))
 
 proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string, extra = "") =
   var err: string
@@ -630,9 +642,10 @@ type
 
 const allExceptModule = {low(TSymKind)..high(TSymKind)} - {skModule, skPackage}
 
-proc lookUpCandidates*(c: PContext, ident: PIdent, filter: set[TSymKind]): seq[PSym] =
+proc lookUpCandidates*(c: PContext, ident: PIdent, filter: set[TSymKind],
+                       includePureEnum = false): seq[PSym] =
   result = searchInScopesFilterBy(c, ident, filter)
-  if result.len == 0:
+  if skEnumField in filter and (result.len == 0 or includePureEnum):
     result.add allPureEnumFields(c, ident)
 
 proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
@@ -665,24 +678,33 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
     c.isAmbiguous = amb
   of nkSym:
     result = n.sym
+  of nkOpenSym:
+    result = qualifiedLookUp(c, n[0], flags)
   of nkDotExpr:
     result = nil
     var m = qualifiedLookUp(c, n[0], (flags * {checkUndeclared}) + {checkModule})
     if m != nil and m.kind == skModule:
       var ident: PIdent = nil
-      if n[1].kind == nkIdent:
-        ident = n[1].ident
-      elif n[1].kind == nkAccQuoted:
+      if n[1].kind == nkAccQuoted:
         ident = considerQuotedIdent(c, n[1])
+      else:
+        # this includes sym and symchoice nodes, but since we are looking in
+        # a module, it shouldn't matter what was captured
+        ident = n[1].getPIdent
       if ident != nil:
         if m == c.module:
-          result = strTableGet(c.topLevelScope.symbols, ident)
+          var ti: TIdentIter = default(TIdentIter)
+          result = initIdentIter(ti, c.topLevelScope.symbols, ident)
+          if result != nil and nextIdentIter(ti, c.topLevelScope.symbols) != nil:
+            # another symbol exists with same name
+            c.isAmbiguous = true
         else:
+          var amb: bool = false
           if c.importModuleLookup.getOrDefault(m.name.id).len > 1:
-            var amb: bool = false
             result = errorUseQualifier(c, n.info, m, amb)
           else:
-            result = someSym(c.graph, m, ident)
+            result = someSymAmb(c.graph, m, ident, amb)
+            if amb: c.isAmbiguous = true
         if result == nil and checkUndeclared in flags:
           result = errorUndeclaredIdentifierHint(c, ident, n[1].info)
       elif n[1].kind == nkSym:
@@ -701,6 +723,10 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
     if result != nil and result.kind == skStub: loadStub(result)
 
 proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
+  if n.kind == nkOpenSym:
+    # maybe the logic in semexprs should be mirrored here instead
+    # for now it only seems this is called for `pickSym` in `getTypeIdent` 
+    return initOverloadIter(o, c, n[0])
   o.importIdx = -1
   o.marked = initIntSet()
   case n.kind

compiler/lowerings.nim

@@ -174,12 +174,12 @@ proc rawIndirectAccess*(a: PNode; field: PSym; info: TLineInfo): PNode =
   # returns a[].field as a node
   assert field.kind == skField
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst)[0]
+  deref.typ() = a.typ.skipTypes(abstractInst)[0]
   deref.add a
   result = newNodeI(nkDotExpr, info)
   result.add deref
   result.add newSymNode(field)
-  result.typ = field.typ
+  result.typ() = field.typ
 
 proc rawDirectAccess*(obj, field: PSym): PNode =
   # returns a.field as a node
@@ -187,7 +187,7 @@ proc rawDirectAccess*(obj, field: PSym): PNode =
   result = newNodeI(nkDotExpr, field.info)
   result.add newSymNode(obj)
   result.add newSymNode(field)
-  result.typ = field.typ
+  result.typ() = field.typ
 
 proc lookupInRecord(n: PNode, id: ItemId): PSym =
   result = nil
@@ -250,12 +250,12 @@ proc newDotExpr*(obj, b: PSym): PNode =
   assert field != nil, b.name.s
   result.add newSymNode(obj)
   result.add newSymNode(field)
-  result.typ = field.typ
+  result.typ() = field.typ
 
 proc indirectAccess*(a: PNode, b: ItemId, info: TLineInfo): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).elementType
+  deref.typ() = a.typ.skipTypes(abstractInst).elementType
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   while true:
@@ -273,12 +273,12 @@ proc indirectAccess*(a: PNode, b: ItemId, info: TLineInfo): PNode =
   result = newNodeI(nkDotExpr, info)
   result.add deref
   result.add newSymNode(field)
-  result.typ = field.typ
+  result.typ() = field.typ
 
 proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).elementType
+  deref.typ() = a.typ.skipTypes(abstractInst).elementType
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   let bb = getIdent(cache, b)
@@ -297,7 +297,7 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): P
   result = newNodeI(nkDotExpr, info)
   result.add deref
   result.add newSymNode(field)
-  result.typ = field.typ
+  result.typ() = field.typ
 
 proc getFieldFromObj*(t: PType; v: PSym): PSym =
   assert v.kind != skField
@@ -320,7 +320,7 @@ proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode =
 proc genAddrOf*(n: PNode; idgen: IdGenerator; typeKind = tyPtr): PNode =
   result = newNodeI(nkAddr, n.info, 1)
   result[0] = n
-  result.typ = newType(typeKind, idgen, n.typ.owner)
+  result.typ() = newType(typeKind, idgen, n.typ.owner)
   result.typ.rawAddSon(n.typ)
 
 proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =
@@ -344,18 +344,18 @@ proc callCodegenProc*(g: ModuleGraph; name: string;
     if optionalArgs != nil:
       for i in 1..<optionalArgs.len-2:
         result.add optionalArgs[i]
-    result.typ = sym.typ.returnType
+    result.typ() = sym.typ.returnType
 
 proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
   result = nkIntLit.newIntNode(value)
-  result.typ = getSysType(g, info, tyInt)
+  result.typ() = getSysType(g, info, tyInt)
 
 proc genHigh*(g: ModuleGraph; n: PNode): PNode =
   if skipTypes(n.typ, abstractVar).kind == tyArray:
     result = newIntLit(g, n.info, toInt64(lastOrd(g.config, skipTypes(n.typ, abstractVar))))
   else:
     result = newNodeI(nkCall, n.info, 2)
-    result.typ = getSysType(g, n.info, tyInt)
+    result.typ() = getSysType(g, n.info, tyInt)
     result[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
     result[1] = n
 
@@ -364,7 +364,7 @@ proc genLen*(g: ModuleGraph; n: PNode): PNode =
     result = newIntLit(g, n.info, toInt64(lastOrd(g.config, skipTypes(n.typ, abstractVar)) + 1))
   else:
     result = newNodeI(nkCall, n.info, 2)
-    result.typ = getSysType(g, n.info, tyInt)
+    result.typ() = getSysType(g, n.info, tyInt)
     result[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
     result[1] = n
 

compiler/magicsys.nim

@@ -166,4 +166,4 @@ proc makeAddr*(n: PNode; idgen: IdGenerator): PNode =
     result = n
   else:
     result = newTree(nkHiddenAddr, n)
-    result.typ = makePtrType(n.typ, idgen)
+    result.typ() = makePtrType(n.typ, idgen)

compiler/main.nim

@@ -22,8 +22,6 @@ import
   modules,
   modulegraphs, lineinfos, pathutils, vmprofiler
 
-# ensure NIR compiles:
-import nir / nir
 
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions]
@@ -48,9 +46,6 @@ proc writeDepsFile(g: ModuleGraph) =
       f.writeLine(toFullPath(g.config, k))
   f.close()
 
-proc writeNinjaFile(g: ModuleGraph) =
-  discard "to implement"
-
 proc writeCMakeDepsFile(conf: ConfigRef) =
   ## write a list of C files for build systems like CMake.
   ## only updated when the C file list changes.
@@ -161,26 +156,6 @@ proc commandCompileToC(graph: ModuleGraph) =
     if optGenCDeps in graph.config.globalOptions:
       writeCMakeDepsFile(conf)
 
-proc commandCompileToNir(graph: ModuleGraph) =
-  let conf = graph.config
-  extccomp.initVars(conf)
-  if conf.symbolFiles == disabledSf:
-    if {optRun, optForceFullMake} * conf.globalOptions == {optRun}:
-      if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile):
-        # nothing changed
-        graph.config.notes = graph.config.mainPackageNotes
-        return
-
-  if not extccomp.ccHasSaneOverflow(conf):
-    conf.symbols.defineSymbol("nimEmulateOverflowChecks")
-
-  if conf.symbolFiles == disabledSf:
-    setPipeLinePass(graph, NirPass)
-  else:
-    setPipeLinePass(graph, SemPass)
-  compilePipelineProject(graph)
-  writeNinjaFile(graph)
-
 proc commandJsonScript(graph: ModuleGraph) =
   extccomp.runJsonBuildInstructions(graph.config, graph.config.jsonBuildInstructionsFile)
 
@@ -197,16 +172,13 @@ proc commandCompileToJS(graph: ModuleGraph) =
     if optGenScript in conf.globalOptions:
       writeDepsFile(graph)
 
-proc commandInteractive(graph: ModuleGraph; useNir: bool) =
+proc commandInteractive(graph: ModuleGraph) =
   graph.config.setErrorMaxHighMaybe
   initDefines(graph.config.symbols)
-  if useNir:
-    defineSymbol(graph.config.symbols, "noSignalHandler")
-  else:
-    defineSymbol(graph.config.symbols, "nimscript")
+  defineSymbol(graph.config.symbols, "nimscript")
   # note: seems redundant with -d:nimHasLibFFI
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
-  setPipeLinePass(graph, if useNir: NirReplPass else: InterpreterPass)
+  setPipeLinePass(graph, InterpreterPass)
   compilePipelineSystemModule(graph)
   if graph.config.commandArgs.len > 0:
     discard graph.compilePipelineModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
@@ -293,8 +265,6 @@ proc mainCommand*(graph: ModuleGraph) =
         # and it has added this define implictly, so we must undo that here.
         # A better solution might be to fix system.nim
         undefSymbol(conf.symbols, "useNimRtl")
-    of backendNir:
-      if conf.exc == excNone: conf.exc = excGoto
     of backendInvalid: raiseAssert "unreachable"
 
   proc compileToBackend() =
@@ -305,7 +275,6 @@ proc mainCommand*(graph: ModuleGraph) =
     of backendCpp: commandCompileToC(graph)
     of backendObjc: commandCompileToC(graph)
     of backendJs: commandCompileToJS(graph)
-    of backendNir: commandCompileToNir(graph)
     of backendInvalid: raiseAssert "unreachable"
 
   template docLikeCmd(body) =
@@ -444,7 +413,7 @@ proc mainCommand*(graph: ModuleGraph) =
     wantMainModule(conf)
     commandView(graph)
     #msgWriteln(conf, "Beware: Indentation tokens depend on the parser's state!")
-  of cmdInteractive: commandInteractive(graph, isDefined(conf, "nir"))
+  of cmdInteractive: commandInteractive(graph)
   of cmdNimscript:
     if conf.projectIsCmd or conf.projectIsStdin: discard
     elif not fileExists(conf.projectFull):

compiler/modulegraphs.nim

@@ -11,7 +11,7 @@
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
 
-import std/[intsets, tables, hashes, strtabs, algorithm]
+import std/[intsets, tables, hashes, strtabs, algorithm, os, strutils, parseutils]
 import ../dist/checksums/src/checksums/md5
 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb
 import ic / [packed_ast, ic]
@@ -62,8 +62,6 @@ type
     CgenPass
     EvalPass
     InterpreterPass
-    NirPass
-    NirReplPass
     GenDependPass
     Docgen2TexPass
     Docgen2JsonPass
@@ -90,6 +88,7 @@ type
     suggestMode*: bool # whether we are in nimsuggest mode or not.
     invalidTransitiveClosure: bool
     interactive*: bool
+    withinSystem*: bool # in system.nim or a module imported by system.nim
     inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the
                                             # first module that included it
     importStack*: seq[FileIndex]  # The current import stack. Used for detecting recursive
@@ -276,6 +275,25 @@ proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
   else:
     result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name)
 
+proc someSymAmb*(g: ModuleGraph; m: PSym; name: PIdent; amb: var bool): PSym =
+  let importHidden = optImportHidden in m.options
+  if isCachedModule(g, m):
+    result = nil
+    for s in interfaceSymbols(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden):
+      if result == nil:
+        # set result to the first symbol
+        result = s
+      else:
+        # another symbol found
+        amb = true
+        break
+  else:
+    var ti: TIdentIter = default(TIdentIter)
+    result = initIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden), name)
+    if result != nil and nextIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden)) != nil:
+      # another symbol exists with same name
+      amb = true
+
 proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym =
   result = someSym(g, g.systemModule, name)
 
@@ -311,7 +329,7 @@ proc resolveInst(g: ModuleGraph; t: var LazyInstantiation): PInstantiation =
     t.inst = result
   assert result != nil
 
-proc resolveAttachedOp(g: ModuleGraph; t: var LazySym): PSym =
+proc resolveAttachedOp*(g: ModuleGraph; t: var LazySym): PSym =
   result = t.sym
   if result == nil:
     result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed)
@@ -454,6 +472,49 @@ proc createMagic*(g: ModuleGraph; idgen: IdGenerator; name: string, m: TMagic):
 proc createMagic(g: ModuleGraph; name: string, m: TMagic): PSym =
   result = createMagic(g, g.idgen, name, m)
 
+proc uniqueModuleName*(conf: ConfigRef; m: PSym): string =
+  ## The unique module name is guaranteed to only contain {'A'..'Z', 'a'..'z', '0'..'9', '_'}
+  ## so that it is useful as a C identifier snippet.
+  let fid = FileIndex(m.position)
+  let path = AbsoluteFile toFullPath(conf, fid)
+  var isLib = false
+  var rel = ""
+  if path.string.startsWith(conf.libpath.string):
+    isLib = true
+    rel = relativeTo(path, conf.libpath).string
+  else:
+    rel = relativeTo(path, conf.projectPath).string
+
+  if not isLib and not belongsToProjectPackage(conf, m):
+    # special handlings for nimble packages
+    when DirSep == '\\':
+      let rel2 = replace(rel, '\\', '/')
+    else:
+      let rel2 = rel
+    const pkgs2 = "pkgs2/"
+    var start = rel2.find(pkgs2)
+    if start >= 0:
+      start += pkgs2.len
+      start += skipUntil(rel2, {'/'}, start)
+      if start+1 < rel2.len:
+        rel = "pkg/" & rel2[start+1..<rel.len] # strips paths
+
+  let trunc = if rel.endsWith(".nim"): rel.len - len(".nim") else: rel.len
+  result = newStringOfCap(trunc)
+  for i in 0..<trunc:
+    let c = rel[i]
+    case c
+    of 'a'..'z', '0'..'9':
+      result.add c
+    of {os.DirSep, os.AltSep}:
+      result.add 'Z' # because it looks a bit like '/'
+    of '.':
+      result.add 'O' # a circle
+    else:
+      # We mangle upper letters too so that there cannot
+      # be clashes with our special meanings of 'Z' and 'O'
+      result.addInt ord(c)
+
 proc registerModule*(g: ModuleGraph; m: PSym) =
   assert m != nil
   assert m.kind == skModule
@@ -465,7 +526,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) =
     setLen(g.packed.pm, m.position + 1)
 
   g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[],
-                               uniqueName: rope(uniqueModuleName(g.config, FileIndex(m.position))))
+                               uniqueName: rope(uniqueModuleName(g.config, m)))
   initStrTables(g, m)
 
 proc registerModuleById*(g: ModuleGraph; m: FileIndex) =
@@ -617,7 +678,6 @@ proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   if m != nil:
     g.suggestSymbols.del(fileIdx)
     g.suggestErrors.del(fileIdx)
-    g.resetForBackend
     incl m.flags, sfDirty
 
 proc unmarkAllDirty*(g: ModuleGraph) =
@@ -680,8 +740,6 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex;
 proc configComplete*(g: ModuleGraph) =
   rememberStartupConfig(g.startupPackedConfig, g.config)
 
-from std/strutils import repeat, `%`
-
 proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) =
   let conf = graph.config
   let isNimscript = conf.isDefined("nimscript")

compiler/modules.nim

@@ -25,7 +25,7 @@ template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
 
 proc partialInitModule*(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
   let packSym = getPackage(graph, fileIdx)
-  result.owner = packSym
+  setOwner(result, packSym)
   result.position = int fileIdx
 
 proc newModule*(graph: ModuleGraph; fileIdx: FileIndex): PSym =

compiler/msgs.nim

@@ -629,7 +629,7 @@ proc warningDeprecated*(conf: ConfigRef, info: TLineInfo = gCmdLineInfo, msg = "
   message(conf, info, warnDeprecated, msg)
 
 proc internalErrorImpl(conf: ConfigRef; info: TLineInfo, errMsg: string, info2: InstantiationInfo) =
-  if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
+  if conf.cmd in {cmdIdeTools, cmdCheck} and conf.structuredErrorHook.isNil: return
   writeContext(conf, info)
   liMessage(conf, info, errInternal, errMsg, doAbort, info2)
 
@@ -651,13 +651,16 @@ template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, extraM
   let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName
   liMessage(conf, info, msg, m, doNothing, instLoc())
 
-proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
-  if i.fileIndex.int32 < 0:
+proc quotedFilename*(conf: ConfigRef; fi: FileIndex): Rope =
+  if fi.int32 < 0:
     result = makeCString "???"
   elif optExcessiveStackTrace in conf.globalOptions:
-    result = conf.m.fileInfos[i.fileIndex.int32].quotedFullName
+    result = conf.m.fileInfos[fi.int32].quotedFullName
   else:
-    result = conf.m.fileInfos[i.fileIndex.int32].quotedName
+    result = conf.m.fileInfos[fi.int32].quotedName
+
+proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
+  quotedFilename(conf, i.fileIndex)
 
 template listMsg(title, r) =
   msgWriteln(conf, title, {msgNoUnitSep})
@@ -666,31 +669,6 @@ template listMsg(title, r) =
 proc listWarnings*(conf: ConfigRef) = listMsg("Warnings:", warnMin..warnMax)
 proc listHints*(conf: ConfigRef) = listMsg("Hints:", hintMin..hintMax)
 
-proc uniqueModuleName*(conf: ConfigRef; fid: FileIndex): string =
-  ## The unique module name is guaranteed to only contain {'A'..'Z', 'a'..'z', '0'..'9', '_'}
-  ## so that it is useful as a C identifier snippet.
-  let path = AbsoluteFile toFullPath(conf, fid)
-  let rel =
-    if path.string.startsWith(conf.libpath.string):
-      relativeTo(path, conf.libpath).string
-    else:
-      relativeTo(path, conf.projectPath).string
-  let trunc = if rel.endsWith(".nim"): rel.len - len(".nim") else: rel.len
-  result = newStringOfCap(trunc)
-  for i in 0..<trunc:
-    let c = rel[i]
-    case c
-    of 'a'..'z':
-      result.add c
-    of {os.DirSep, os.AltSep}:
-      result.add 'Z' # because it looks a bit like '/'
-    of '.':
-      result.add 'O' # a circle
-    else:
-      # We mangle upper letters and digits too so that there cannot
-      # be clashes with our special meanings of 'Z' and 'O'
-      result.addInt ord(c)
-
 proc genSuccessX*(conf: ConfigRef) =
   let mem =
     when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"

compiler/ndi.nim

@@ -1,52 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2017 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements the generation of ``.ndi`` files for better debugging
-## support of Nim code. "ndi" stands for "Nim debug info".
-
-import ast, msgs, ropes, options, pathutils
-
-when defined(nimPreviewSlimSystem):
-  import std/[syncio, assertions]
-
-type
-  NdiFile* = object
-    enabled: bool
-    f: File
-    buf: string
-    filename: AbsoluteFile
-    syms: seq[PSym]
-
-proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
-  f.buf.setLen 0
-  f.buf.addInt s.info.line.int
-  f.buf.add "\t"
-  f.buf.addInt s.info.col.int
-  f.f.write(s.name.s, "\t")
-  f.f.writeRope(s.loc.r)
-  f.f.writeLine("\t", toFullPath(conf, s.info), "\t", f.buf)
-
-template writeMangledName*(f: NdiFile; s: PSym; conf: ConfigRef) =
-  if f.enabled: f.syms.add s
-
-proc open*(f: var NdiFile; filename: AbsoluteFile; conf: ConfigRef) =
-  f.enabled = not filename.isEmpty
-  if f.enabled:
-    f.filename = filename
-    f.buf = newStringOfCap(20)
-
-proc close*(f: var NdiFile, conf: ConfigRef) =
-  if f.enabled:
-    f.f = open(f.filename.string, fmWrite, 8000)
-    doAssert f.f != nil, f.filename.string
-    for s in f.syms:
-      doWrite(f, s, conf)
-    close(f.f)
-    f.syms.reset
-    f.filename.reset

compiler/nilcheck.nim

@@ -919,7 +919,7 @@ proc infix(ctx: NilCheckerContext, l: PNode, r: PNode, magic: TMagic): PNode =
     newSymNode(op, r.info),
     l,
     r)
-  result.typ = newType(tyBool, ctx.idgen, nil)
+  result.typ() = newType(tyBool, ctx.idgen, nil)
 
 proc prefixNot(ctx: NilCheckerContext, node: PNode): PNode =
   var cache = newIdentCache()
@@ -929,7 +929,7 @@ proc prefixNot(ctx: NilCheckerContext, node: PNode): PNode =
   result = nkPrefix.newTree(
     newSymNode(op, node.info),
     node)
-  result.typ = newType(tyBool, ctx.idgen, nil)
+  result.typ() = newType(tyBool, ctx.idgen, nil)
 
 proc infixEq(ctx: NilCheckerContext, l: PNode, r: PNode): PNode =
   infix(ctx, l, r, mEqRef)

compiler/nim.cfg

@@ -4,7 +4,6 @@ hint[XDeclaredButNotUsed]:off
 
 define:booting
 define:nimcore
-define:nimPreviewFloatRoundtrip
 define:nimPreviewSlimSystem
 define:nimPreviewCstringConversion
 define:nimPreviewProcConversion

compiler/nim.nim

@@ -116,9 +116,9 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
     conf.backend = backendC
 
   if conf.selectedGC == gcUnselected:
-    if conf.backend in {backendC, backendCpp, backendObjc, backendNir} or
-        (conf.cmd == cmdInteractive and isDefined(conf, "nir")) or
-        (conf.cmd in cmdDocLike and conf.backend != backendJs):
+    if conf.backend in {backendC, backendCpp, backendObjc} or
+        (conf.cmd in cmdDocLike and conf.backend != backendJs) or
+        conf.cmd == cmdGendepend:
       initOrcDefines(conf)
 
   mainCommand(graph)

compiler/nimsets.nim

@@ -84,7 +84,7 @@ proc toTreeSet*(conf: ConfigRef; s: TBitSet, settype: PType, info: TLineInfo): P
   elemType = settype[0]
   first = firstOrd(conf, elemType).toInt64
   result = newNodeI(nkCurly, info)
-  result.typ = settype
+  result.typ() = settype
   result.info = info
   e = 0
   while e < s.len * ElemSize:
@@ -101,7 +101,7 @@ proc toTreeSet*(conf: ConfigRef; s: TBitSet, settype: PType, info: TLineInfo): P
         result.add aa
       else:
         n = newNodeI(nkRange, info)
-        n.typ = elemType
+        n.typ() = elemType
         n.add aa
         let bb = newIntTypeNode(b + first, elemType)
         bb.info = info

compiler/nir/cir.nim

@@ -1,983 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-# We produce C code as a list of tokens.
-
-import std / [assertions, syncio, tables, intsets, formatfloat]
-from std / strutils import toOctal
-import .. / ic / [bitabs, rodfiles]
-import nirtypes, nirinsts, nirfiles
-import ../../dist/checksums/src/checksums/md5
-
-type
-  Token = LitId # indexing into the tokens BiTable[string]
-
-  PredefinedToken = enum
-    IgnoreMe = "<unused>"
-    EmptyToken = ""
-    CurlyLe = "{"
-    CurlyRi = "}"
-    ParLe = "("
-    ParRi = ")"
-    BracketLe = "["
-    BracketRi = "]"
-    NewLine = "\n"
-    Semicolon = ";"
-    Comma = ", "
-    Space = " "
-    Colon = ": "
-    Dot = "."
-    Arrow = "->"
-    Star = "*"
-    Amp = "&"
-    AsgnOpr = " = "
-    ScopeOpr = "::"
-    ConstKeyword = "const "
-    StaticKeyword = "static "
-    ExternKeyword = "extern "
-    WhileKeyword = "while "
-    IfKeyword = "if ("
-    ElseKeyword = "else "
-    SwitchKeyword = "switch "
-    CaseKeyword = "case "
-    DefaultKeyword = "default:"
-    BreakKeyword = "break"
-    NullPtr = "nullptr"
-    IfNot = "if (!("
-    ReturnKeyword = "return "
-    TypedefStruct = "typedef struct "
-    TypedefUnion = "typedef union "
-    IncludeKeyword = "#include "
-
-proc fillTokenTable(tab: var BiTable[string]) =
-  for e in EmptyToken..high(PredefinedToken):
-    let id = tab.getOrIncl $e
-    assert id == LitId(e), $(id, " ", ord(e))
-
-type
-  GeneratedCode* = object
-    m: NirModule
-    includes: seq[LitId]
-    includedHeaders: IntSet
-    data: seq[LitId]
-    protos: seq[LitId]
-    code: seq[LitId]
-    init: seq[LitId]
-    tokens: BiTable[string]
-    emittedStrings: IntSet
-    needsPrefix: IntSet
-    generatedTypes: IntSet
-    mangledModules: Table[LitId, LitId]
-
-proc initGeneratedCode*(m: sink NirModule): GeneratedCode =
-  result = GeneratedCode(m: m, code: @[], tokens: initBiTable[string]())
-  fillTokenTable(result.tokens)
-
-proc add*(g: var GeneratedCode; t: PredefinedToken) {.inline.} =
-  g.code.add Token(t)
-
-proc add*(g: var GeneratedCode; s: string) {.inline.} =
-  g.code.add g.tokens.getOrIncl(s)
-
-proc mangleModuleName(c: var GeneratedCode; key: LitId): LitId =
-  result = c.mangledModules.getOrDefault(key, LitId(0))
-  if result == LitId(0):
-    let u {.cursor.} = c.m.lit.strings[key]
-    var last = u.len - len(".nim") - 1
-    var start = last
-    while start >= 0 and u[start] != '/': dec start
-    var sum = getMD5(u)
-    sum.setLen(8)
-    let dest = u.substr(start+1, last) & sum
-    result = c.tokens.getOrIncl(dest)
-    c.mangledModules[key] = result
-
-type
-  CppFile = object
-    f: File
-
-proc write(f: var CppFile; s: string) = write(f.f, s)
-proc write(f: var CppFile; c: char) = write(f.f, c)
-
-proc writeTokenSeq(f: var CppFile; s: seq[Token]; c: GeneratedCode) =
-  var indent = 0
-  for i in 0..<s.len:
-    let x = s[i]
-    case x
-    of Token(CurlyLe):
-      inc indent
-      write f, c.tokens[x]
-      write f, "\n"
-      for i in 1..indent*2: write f, ' '
-    of Token(CurlyRi):
-      dec indent
-      write f, c.tokens[x]
-      if i+1 < s.len and s[i+1] == Token(CurlyRi):
-        discard
-      else:
-        write f, "\n"
-        for i in 1..indent*2: write f, ' '
-    of Token(Semicolon):
-      write f, c.tokens[x]
-      if i+1 < s.len and s[i+1] == Token(CurlyRi):
-        discard "no newline before }"
-      else:
-        write f, "\n"
-        for i in 1..indent*2: write f, ' '
-    of Token(NewLine):
-      write f, c.tokens[x]
-      for i in 1..indent*2: write f, ' '
-    else:
-      write f, c.tokens[x]
-
-
-# Type graph
-
-type
-  TypeList = object
-    processed: IntSet
-    s: seq[(TypeId, PredefinedToken)]
-
-proc add(dest: var TypeList; elem: TypeId; decl: PredefinedToken) =
-  if not containsOrIncl(dest.processed, int(elem)):
-    dest.s.add (elem, decl)
-
-type
-  TypeOrder = object
-    forwardedDecls, ordered: TypeList
-    typeImpls: Table[string, TypeId]
-    lookedAt: IntSet
-
-proc traverseObject(types: TypeGraph; lit: Literals; c: var TypeOrder; t: TypeId)
-
-proc recordDependency(types: TypeGraph; lit: Literals; c: var TypeOrder; parent, child: TypeId) =
-  var ch = child
-  var viaPointer = false
-  while true:
-    case types[ch].kind
-    of APtrTy, UPtrTy, AArrayPtrTy, UArrayPtrTy:
-      viaPointer = true
-      ch = elementType(types, ch)
-    of LastArrayTy:
-      ch = elementType(types, ch)
-    else:
-      break
-
-  case types[ch].kind
-  of ObjectTy, UnionTy:
-    let decl = if types[ch].kind == ObjectTy: TypedefStruct else: TypedefUnion
-    let obj = c.typeImpls.getOrDefault(lit.strings[types[ch].litId])
-    if viaPointer:
-      c.forwardedDecls.add obj, decl
-    else:
-      if not containsOrIncl(c.lookedAt, obj.int):
-        traverseObject(types, lit, c, obj)
-      c.ordered.add obj, decl
-  of ArrayTy:
-    if viaPointer:
-      c.forwardedDecls.add ch, TypedefStruct
-    else:
-      if not containsOrIncl(c.lookedAt, ch.int):
-        traverseObject(types, lit, c, ch)
-      c.ordered.add ch, TypedefStruct
-  else:
-    discard "uninteresting type as we only focus on the required struct declarations"
-
-proc traverseObject(types: TypeGraph; lit: Literals; c: var TypeOrder; t: TypeId) =
-  for x in sons(types, t):
-    case types[x].kind
-    of FieldDecl:
-      recordDependency types, lit, c, t, x.firstSon
-    of ObjectTy:
-      # inheritance
-      recordDependency types, lit, c, t, x
-    else: discard
-
-proc traverseTypes(types: TypeGraph; lit: Literals; c: var TypeOrder) =
-  for t in allTypes(types):
-    if types[t].kind in {ObjectDecl, UnionDecl}:
-      assert types[t.firstSon].kind == NameVal
-      c.typeImpls[lit.strings[types[t.firstSon].litId]] = t
-
-  for t in allTypesIncludingInner(types):
-    case types[t].kind
-    of ObjectDecl, UnionDecl:
-      traverseObject types, lit, c, t
-      let decl = if types[t].kind == ObjectDecl: TypedefStruct else: TypedefUnion
-      c.ordered.add t, decl
-    of ArrayTy:
-      traverseObject types, lit, c, t
-      c.ordered.add t, TypedefStruct
-    else: discard
-
-when false:
-  template emitType(s: string) = c.types.add c.tokens.getOrIncl(s)
-  template emitType(t: Token) = c.types.add t
-  template emitType(t: PredefinedToken) = c.types.add Token(t)
-
-proc genType(g: var GeneratedCode; types: TypeGraph; lit: Literals; t: TypeId; name = "") =
-  template maybeAddName =
-    if name != "":
-      g.add Space
-      g.add name
-
-  template atom(s: string) =
-    g.add s
-    maybeAddName()
-  case types[t].kind
-  of VoidTy: atom "void"
-  of IntTy: atom "NI" & $types[t].integralBits
-  of UIntTy: atom "NU" & $types[t].integralBits
-  of FloatTy: atom "NF" & $types[t].integralBits
-  of BoolTy: atom "NB" & $types[t].integralBits
-  of CharTy: atom "NC" & $types[t].integralBits
-  of ObjectTy, UnionTy, NameVal, AnnotationVal:
-    atom lit.strings[types[t].litId]
-  of VarargsTy:
-    g.add "..."
-  of APtrTy, UPtrTy, AArrayPtrTy, UArrayPtrTy:
-    genType g, types, lit, elementType(types, t)
-    g.add Star
-    maybeAddName()
-  of ArrayTy:
-    genType g, types, lit, arrayName(types, t)
-    maybeAddName()
-  of LastArrayTy:
-    genType g, types, lit, elementType(types, t)
-    maybeAddName()
-    g.add BracketLe
-    g.add BracketRi
-  of ProcTy:
-    let (retType, callConv) = returnType(types, t)
-    genType g, types, lit, retType
-    g.add Space
-    g.add ParLe
-    genType g, types, lit, callConv
-    g.add Star # "(*fn)"
-    maybeAddName()
-    g.add ParRi
-    g.add ParLe
-    var i = 0
-    for ch in params(types, t):
-      if i > 0: g.add Comma
-      genType g, types, lit, ch
-      inc i
-    g.add ParRi
-  of ObjectDecl, UnionDecl:
-    atom lit.strings[types[t.firstSon].litId]
-  of IntVal, SizeVal, AlignVal, OffsetVal, FieldDecl:
-    #raiseAssert "did not expect: " & $types[t].kind
-    g.add "BUG "
-    atom $types[t].kind
-
-proc generateTypes(g: var GeneratedCode; types: TypeGraph; lit: Literals; c: TypeOrder) =
-  for (t, declKeyword) in c.forwardedDecls.s:
-    let name = if types[t].kind == ArrayTy: arrayName(types, t) else: t.firstSon
-    let s {.cursor.} = lit.strings[types[name].litId]
-    g.add declKeyword
-    g.add s
-    g.add Space
-    g.add s
-    g.add Semicolon
-
-  for (t, declKeyword) in c.ordered.s:
-    let name = if types[t].kind == ArrayTy: arrayName(types, t) else: t.firstSon
-    let litId = types[name].litId
-    if not g.generatedTypes.containsOrIncl(litId.int):
-      let s {.cursor.} = lit.strings[litId]
-      g.add declKeyword
-      g.add CurlyLe
-      if types[t].kind == ArrayTy:
-        genType g, types, lit, elementType(types, t), "a"
-        g.add BracketLe
-        g.add $arrayLen(types, t)
-        g.add BracketRi
-        g.add Semicolon
-      else:
-        var i = 0
-        for x in sons(types, t):
-          case types[x].kind
-          of FieldDecl:
-            genType g, types, lit, x.firstSon, "F" & $i
-            g.add Semicolon
-            inc i
-          of ObjectTy:
-            genType g, types, lit, x, "P"
-            g.add Semicolon
-          else: discard
-      g.add CurlyRi
-      g.add s
-      g.add Semicolon
-
-# Procs
-
-proc toCChar*(c: char; result: var string) {.inline.} =
-  case c
-  of '\0'..'\x1F', '\x7F'..'\xFF':
-    result.add '\\'
-    result.add toOctal(c)
-  of '\'', '\"', '\\', '?':
-    result.add '\\'
-    result.add c
-  else:
-    result.add c
-
-proc makeCString(s: string): string =
-  result = newStringOfCap(s.len + 10)
-  result.add('"')
-  for c in s: toCChar(c, result)
-  result.add('"')
-
-template emitData(s: string) = c.data.add c.tokens.getOrIncl(s)
-template emitData(t: Token) = c.data.add t
-template emitData(t: PredefinedToken) = c.data.add Token(t)
-
-proc genStrLit(c: var GeneratedCode; lit: Literals; litId: LitId): Token =
-  result = Token(c.tokens.getOrIncl "QStr" & $litId)
-  if not containsOrIncl(c.emittedStrings, int(litId)):
-    let s {.cursor.} = lit.strings[litId]
-    emitData "static const struct "
-    emitData CurlyLe
-    emitData "NI cap"
-    emitData Semicolon
-    emitData "NC8 data"
-    emitData BracketLe
-    emitData $s.len
-    emitData "+1"
-    emitData BracketRi
-    emitData Semicolon
-    emitData CurlyRi
-    emitData result
-    emitData AsgnOpr
-    emitData CurlyLe
-    emitData $s.len
-    emitData " | NIM_STRLIT_FLAG"
-    emitData Comma
-    emitData makeCString(s)
-    emitData CurlyRi
-    emitData Semicolon
-
-proc genIntLit(c: var GeneratedCode; lit: Literals; litId: LitId) =
-  let i = lit.numbers[litId]
-  if i > low(int32) and i <= high(int32):
-    c.add $i
-  elif i == low(int32):
-    # Nim has the same bug for the same reasons :-)
-    c.add "(-2147483647 -1)"
-  elif i > low(int64):
-    c.add "IL64("
-    c.add $i
-    c.add ")"
-  else:
-    c.add "(IL64(-9223372036854775807) - IL64(1))"
-
-proc gen(c: var GeneratedCode; t: Tree; n: NodePos)
-
-proc genDisplayName(c: var GeneratedCode; symId: SymId) =
-  let displayName = c.m.symnames[symId]
-  if displayName != LitId(0):
-    c.add "/*"
-    c.add c.m.lit.strings[displayName]
-    c.add "*/"
-
-proc genSymDef(c: var GeneratedCode; t: Tree; n: NodePos) =
-  if t[n].kind == SymDef:
-    let symId = t[n].symId
-    c.needsPrefix.incl symId.int
-    genDisplayName c, symId
-  gen c, t, n
-
-proc genGlobal(c: var GeneratedCode; t: Tree; name, typ: NodePos; annotation: string) =
-  c.add annotation
-  let m: string
-  if t[name].kind == SymDef:
-    let symId = t[name].symId
-    m = c.tokens[mangleModuleName(c, c.m.namespace)] & "__" & $symId
-    genDisplayName c, symId
-  else:
-    assert t[name].kind == ModuleSymUse
-    let (x, y) = sons2(t, name)
-    m = c.tokens[mangleModuleName(c, t[x].litId)] & "__" & $t[y].immediateVal
-  genType c, c.m.types, c.m.lit, t[typ].typeId, m
-
-proc genLocal(c: var GeneratedCode; t: Tree; name, typ: NodePos; annotation: string) =
-  assert t[name].kind == SymDef
-  c.add annotation
-  let symId = t[name].symId
-  genType c, c.m.types, c.m.lit, t[typ].typeId, "q" & $symId
-  genDisplayName c, symId
-
-proc genProcDecl(c: var GeneratedCode; t: Tree; n: NodePos; isExtern: bool) =
-  let signatureBegin = c.code.len
-  let name = n.firstSon
-
-  var prc = n.firstSon
-  next t, prc
-
-  while true:
-    case t[prc].kind
-    of PragmaPair:
-      let (x, y) = sons2(t, prc)
-      let key = cast[PragmaKey](t[x].rawOperand)
-      case key
-      of HeaderImport:
-        let lit = t[y].litId
-        let headerAsStr {.cursor.} = c.m.lit.strings[lit]
-        let header = c.tokens.getOrIncl(headerAsStr)
-        # headerAsStr can be empty, this has the semantics of the `nodecl` pragma:
-        if headerAsStr.len > 0 and not c.includedHeaders.containsOrIncl(int header):
-          if headerAsStr[0] == '#':
-            discard "skip the #include"
-          else:
-            c.includes.add Token(IncludeKeyword)
-          c.includes.add header
-          c.includes.add Token NewLine
-        # do not generate code for importc'ed procs:
-        return
-      of DllImport:
-        let lit = t[y].litId
-        raiseAssert "cannot eval: " & c.m.lit.strings[lit]
-      else: discard
-    of PragmaId: discard
-    else: break
-    next t, prc
-
-  var resultDeclPos = NodePos(-1)
-  if t[prc].kind == SummonResult:
-    resultDeclPos = prc
-    gen c, t, prc.firstSon
-    next t, prc
-  else:
-    c.add "void"
-  c.add Space
-  genSymDef c, t, name
-  c.add ParLe
-  var params = 0
-  while t[prc].kind == SummonParam:
-    if params > 0: c.add Comma
-    let (typ, sym) = sons2(t, prc)
-    genLocal c, t, sym, typ, ""
-    next t, prc
-    inc params
-  if params == 0:
-    c.add "void"
-  c.add ParRi
-
-  for i in signatureBegin ..< c.code.len:
-    c.protos.add c.code[i]
-  c.protos.add Token Semicolon
-
-  if isExtern:
-    c.code.setLen signatureBegin
-  else:
-    c.add CurlyLe
-    if resultDeclPos.int >= 0:
-      gen c, t, resultDeclPos
-    for ch in sonsRest(t, n, prc):
-      gen c, t, ch
-    c.add CurlyRi
-
-template triop(opr) =
-  let (typ, a, b) = sons3(t, n)
-  c.add ParLe
-  c.add ParLe
-  gen c, t, typ
-  c.add ParRi
-  gen c, t, a
-  c.add opr
-  gen c, t, b
-  c.add ParRi
-
-template cmpop(opr) =
-  let (_, a, b) = sons3(t, n)
-  c.add ParLe
-  gen c, t, a
-  c.add opr
-  gen c, t, b
-  c.add ParRi
-
-template binaryop(opr) =
-  let (typ, a) = sons2(t, n)
-  c.add ParLe
-  c.add ParLe
-  gen c, t, typ
-  c.add ParRi
-  c.add opr
-  gen c, t, a
-  c.add ParRi
-
-template checkedBinaryop(opr) =
-  let (typ, labIdx, a, b) = sons4(t, n)
-  let bits = integralBits(c.m.types[t[typ].typeId])
-  let lab = t[labIdx].label
-
-  c.add (opr & $bits)
-  c.add ParLe
-  c.gen t, a
-  c.add Comma
-  c.gen t, b
-  c.add Comma
-  c.add "L" & $lab.int
-  c.add ParRi
-
-proc genNumberConv(c: var GeneratedCode; t: Tree; n: NodePos) =
-  let (typ, arg) = sons2(t, n)
-  if t[arg].kind == IntVal:
-    let litId = t[arg].litId
-    c.add ParLe
-    c.add ParLe
-    gen c, t, typ
-    c.add ParRi
-    case c.m.types[t[typ].typeId].kind
-    of UIntTy:
-      let x = cast[uint64](c.m.lit.numbers[litId])
-      c.add $x
-    of FloatTy:
-      let x = cast[float64](c.m.lit.numbers[litId])
-      c.add $x
-    else:
-      gen c, t, arg
-    c.add ParRi
-  else:
-    binaryop ""
-
-template moveToDataSection(body: untyped) =
-  let oldLen = c.code.len
-  body
-  for i in oldLen ..< c.code.len:
-    c.data.add c.code[i]
-  setLen c.code, oldLen
-
-proc gen(c: var GeneratedCode; t: Tree; n: NodePos) =
-  case t[n].kind
-  of Nop:
-    discard "nothing to emit"
-  of ImmediateVal:
-    c.add $t[n].immediateVal
-  of IntVal:
-    genIntLit c, c.m.lit, t[n].litId
-  of StrVal:
-    c.code.add genStrLit(c, c.m.lit, t[n].litId)
-  of Typed:
-    genType c, c.m.types, c.m.lit, t[n].typeId
-  of SymDef, SymUse:
-    let s = t[n].symId
-    if c.needsPrefix.contains(s.int):
-      c.code.add mangleModuleName(c, c.m.namespace)
-      c.add "__"
-      c.add $s
-    else:
-      # XXX Use proper names here
-      c.add "q"
-      c.add $s
-
-  of ModuleSymUse:
-    let (x, y) = sons2(t, n)
-    let u = mangleModuleName(c, t[x].litId)
-    let s = t[y].immediateVal
-    c.code.add u
-    c.add "__"
-    c.add $s
-
-  of NilVal:
-    c.add NullPtr
-  of LoopLabel:
-    c.add WhileKeyword
-    c.add ParLe
-    c.add "1"
-    c.add ParRi
-    c.add CurlyLe
-  of GotoLoop:
-    c.add CurlyRi
-  of Label:
-    let lab = t[n].label
-    c.add "L"
-    c.add $lab.int
-    c.add Colon
-    c.add Semicolon
-  of Goto:
-    let lab = t[n].label
-    c.add "goto L"
-    c.add $lab.int
-    c.add Semicolon
-  of CheckedGoto:
-    discard "XXX todo"
-  of ArrayConstr:
-    c.add CurlyLe
-    c.add ".a = "
-    c.add CurlyLe
-    var i = 0
-    for ch in sonsFrom1(t, n):
-      if i > 0: c.add Comma
-      c.gen t, ch
-      inc i
-    c.add CurlyRi
-    c.add CurlyRi
-  of ObjConstr:
-    c.add CurlyLe
-    var i = 0
-    for ch in sonsFrom1(t, n):
-      if i mod 2 == 0:
-        if i > 0: c.add Comma
-        c.add ".F" & $t[ch].immediateVal
-        c.add AsgnOpr
-      else:
-        c.gen t, ch
-      inc i
-    c.add CurlyRi
-  of Ret:
-    c.add ReturnKeyword
-    c.gen t, n.firstSon
-    c.add Semicolon
-  of Select:
-    c.add SwitchKeyword
-    c.add ParLe
-    let (_, selector) = sons2(t, n)
-    c.gen t, selector
-    c.add ParRi
-    c.add CurlyLe
-    for ch in sonsFromN(t, n, 2):
-      c.gen t, ch
-    c.add CurlyRi
-  of SelectPair:
-    let (le, ri) = sons2(t, n)
-    c.gen t, le
-    c.gen t, ri
-  of SelectList:
-    for ch in sons(t, n):
-      c.gen t, ch
-  of SelectValue:
-    c.add CaseKeyword
-    c.gen t, n.firstSon
-    c.add Colon
-  of SelectRange:
-    let (le, ri) = sons2(t, n)
-    c.add CaseKeyword
-    c.gen t, le
-    c.add " ... "
-    c.gen t, ri
-    c.add Colon
-  of ForeignDecl:
-    c.data.add LitId(ExternKeyword)
-    c.gen t, n.firstSon
-  of SummonGlobal:
-    moveToDataSection:
-      let (typ, sym) = sons2(t, n)
-      c.genGlobal t, sym, typ, ""
-      c.add Semicolon
-  of SummonThreadLocal:
-    moveToDataSection:
-      let (typ, sym) = sons2(t, n)
-      c.genGlobal t, sym, typ, "__thread "
-      c.add Semicolon
-  of SummonConst:
-    moveToDataSection:
-      let (typ, sym, val) = sons3(t, n)
-      c.genGlobal t, sym, typ, "const "
-      c.add AsgnOpr
-      c.gen t, val
-      c.add Semicolon
-  of Summon, SummonResult:
-    let (typ, sym) = sons2(t, n)
-    c.genLocal t, sym, typ, ""
-    c.add Semicolon
-
-  of SummonParam:
-    raiseAssert "SummonParam should have been handled in genProc"
-  of Kill:
-    discard "we don't care about Kill instructions"
-  of AddrOf:
-    let (_, arg) = sons2(t, n)
-    c.add "&"
-    gen c, t, arg
-  of DerefArrayAt:
-    let (_, a, i) = sons3(t, n)
-    gen c, t, a
-    c.add BracketLe
-    gen c, t, i
-    c.add BracketRi
-  of ArrayAt:
-    let (_, a, i) = sons3(t, n)
-    gen c, t, a
-    c.add Dot
-    c.add "a"
-    c.add BracketLe
-    gen c, t, i
-    c.add BracketRi
-  of FieldAt:
-    let (_, a, b) = sons3(t, n)
-    gen c, t, a
-    let field = t[b].immediateVal
-    c.add Dot
-    c.add "F" & $field
-  of DerefFieldAt:
-    let (_, a, b) = sons3(t, n)
-    gen c, t, a
-    let field = t[b].immediateVal
-    c.add Arrow
-    c.add "F" & $field
-  of Load:
-    let (_, arg) = sons2(t, n)
-    c.add ParLe
-    c.add "*"
-    gen c, t, arg
-    c.add ParRi
-  of Store:
-    raiseAssert "Assumption was that Store is unused!"
-  of Asgn:
-    let (_, dest, src) = sons3(t, n)
-    gen c, t, dest
-    c.add AsgnOpr
-    gen c, t, src
-    c.add Semicolon
-  of CheckedRange:
-    c.add "nimCheckRange"
-    c.add ParLe
-    let (_, gotoInstr, x, a, b) = sons5(t, n)
-    gen c, t, x
-    c.add Comma
-    gen c, t, a
-    c.add Comma
-    gen c, t, b
-    c.add Comma
-    c.add "L" & $t[gotoInstr].label.int
-    c.add ParRi
-  of CheckedIndex:
-    c.add "nimCheckIndex"
-    c.add ParLe
-    let (gotoInstr, x, a) = sons3(t, n)
-    gen c, t, x
-    c.add Comma
-    gen c, t, a
-    c.add Comma
-    c.add "L" & $t[gotoInstr].label.int
-    c.add ParRi
-  of Call, IndirectCall:
-    let (typ, fn) = sons2(t, n)
-    gen c, t, fn
-    c.add ParLe
-    var i = 0
-    for ch in sonsFromN(t, n, 2):
-      if i > 0: c.add Comma
-      gen c, t, ch
-      inc i
-    c.add ParRi
-    if c.m.types[t[typ].typeId].kind == VoidTy:
-      c.add Semicolon
-  of CheckedCall, CheckedIndirectCall:
-    let (typ, gotoInstr, fn) = sons3(t, n)
-    gen c, t, fn
-    c.add ParLe
-    var i = 0
-    for ch in sonsFromN(t, n, 3):
-      if i > 0: c.add Comma
-      gen c, t, ch
-      inc i
-    c.add ParRi
-    if c.m.types[t[typ].typeId].kind == VoidTy:
-      c.add Semicolon
-
-  of CheckedAdd: checkedBinaryop "nimAddInt"
-  of CheckedSub: checkedBinaryop "nimSubInt"
-  of CheckedMul: checkedBinaryop "nimMulInt"
-  of CheckedDiv: checkedBinaryop "nimDivInt"
-  of CheckedMod: checkedBinaryop "nimModInt"
-  of Add: triop " + "
-  of Sub: triop " - "
-  of Mul: triop " * "
-  of Div: triop " / "
-  of Mod: triop " % "
-  of BitShl: triop " << "
-  of BitShr: triop " >> "
-  of BitAnd: triop " & "
-  of BitOr: triop " | "
-  of BitXor: triop " ^ "
-  of BitNot: binaryop " ~ "
-  of BoolNot: binaryop " !"
-  of Eq: cmpop " == "
-  of Le: cmpop " <= "
-  of Lt: cmpop " < "
-  of Cast: binaryop ""
-  of NumberConv: genNumberConv c, t, n
-  of CheckedObjConv: binaryop ""
-  of ObjConv: binaryop ""
-  of Emit: raiseAssert "cannot interpret: Emit"
-  of ProcDecl: genProcDecl c, t, n, false
-  of ForeignProcDecl: genProcDecl c, t, n, true
-  of PragmaPair, PragmaId, TestOf, Yld, SetExc, TestExc:
-    c.add "cannot interpret: " & $t[n].kind
-
-const
-  Prelude = """
-/* GENERATED CODE. DO NOT EDIT. */
-
-#ifdef __cplusplus
-#define NB8 bool
-#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901)
-// see #13798: to avoid conflicts for code emitting `#include <stdbool.h>`
-#define NB8 _Bool
-#else
-typedef unsigned char NB8; // best effort
-#endif
-
-typedef unsigned char NC8;
-
-typedef float NF32;
-typedef double NF64;
-#if defined(__BORLANDC__) || defined(_MSC_VER)
-typedef signed char NI8;
-typedef signed short int NI16;
-typedef signed int NI32;
-typedef __int64 NI64;
-/* XXX: Float128? */
-typedef unsigned char NU8;
-typedef unsigned short int NU16;
-typedef unsigned int NU32;
-typedef unsigned __int64 NU64;
-#elif defined(HAVE_STDINT_H)
-#ifndef USE_NIM_NAMESPACE
-#  include <stdint.h>
-#endif
-typedef int8_t NI8;
-typedef int16_t NI16;
-typedef int32_t NI32;
-typedef int64_t NI64;
-typedef uint8_t NU8;
-typedef uint16_t NU16;
-typedef uint32_t NU32;
-typedef uint64_t NU64;
-#elif defined(HAVE_CSTDINT)
-#ifndef USE_NIM_NAMESPACE
-#  include <cstdint>
-#endif
-typedef std::int8_t NI8;
-typedef std::int16_t NI16;
-typedef std::int32_t NI32;
-typedef std::int64_t NI64;
-typedef std::uint8_t NU8;
-typedef std::uint16_t NU16;
-typedef std::uint32_t NU32;
-typedef std::uint64_t NU64;
-#else
-/* Unknown compiler/version, do our best */
-#ifdef __INT8_TYPE__
-typedef __INT8_TYPE__ NI8;
-#else
-typedef signed char NI8;
-#endif
-#ifdef __INT16_TYPE__
-typedef __INT16_TYPE__ NI16;
-#else
-typedef signed short int NI16;
-#endif
-#ifdef __INT32_TYPE__
-typedef __INT32_TYPE__ NI32;
-#else
-typedef signed int NI32;
-#endif
-#ifdef __INT64_TYPE__
-typedef __INT64_TYPE__ NI64;
-#else
-typedef long long int NI64;
-#endif
-/* XXX: Float128? */
-#ifdef __UINT8_TYPE__
-typedef __UINT8_TYPE__ NU8;
-#else
-typedef unsigned char NU8;
-#endif
-#ifdef __UINT16_TYPE__
-typedef __UINT16_TYPE__ NU16;
-#else
-typedef unsigned short int NU16;
-#endif
-#ifdef __UINT32_TYPE__
-typedef __UINT32_TYPE__ NU32;
-#else
-typedef unsigned int NU32;
-#endif
-#ifdef __UINT64_TYPE__
-typedef __UINT64_TYPE__ NU64;
-#else
-typedef unsigned long long int NU64;
-#endif
-#endif
-
-#ifdef NIM_INTBITS
-#  if NIM_INTBITS == 64
-typedef NI64 NI;
-typedef NU64 NU;
-#  elif NIM_INTBITS == 32
-typedef NI32 NI;
-typedef NU32 NU;
-#  elif NIM_INTBITS == 16
-typedef NI16 NI;
-typedef NU16 NU;
-#  elif NIM_INTBITS == 8
-typedef NI8 NI;
-typedef NU8 NU;
-#  else
-#    error "invalid bit width for int"
-#  endif
-#endif
-
-#define NIM_STRLIT_FLAG ((NU)(1) << ((NIM_INTBITS) - 2)) /* This has to be the same as system.strlitFlag! */
-
-#define nimAddInt64(a, b, L) ({long long int res; if(__builtin_saddll_overflow(a, b, &res)) goto L; res})
-#define nimSubInt64(a, b, L) ({long long int res; if(__builtin_ssubll_overflow(a, b, &res)) goto L; res})
-#define nimMulInt64(a, b, L) ({long long int res; if(__builtin_smulll_overflow(a, b, &res)) goto L; res})
-
-#define nimAddInt32(a, b, L) ({long int res; if(__builtin_sadd_overflow(a, b, &res)) goto L; res})
-#define nimSubInt32(a, b, L) ({long int res; if(__builtin_ssub_overflow(a, b, &res)) goto L; res})
-#define nimMulInt32(a, b, L) ({long int res; if(__builtin_smul_overflow(a, b, &res)) goto L; res})
-
-#define nimCheckRange(x, a, b, L) ({if (x < a || x > b) goto L; x})
-#define nimCheckIndex(x, a, L) ({if (x >= a) goto L; x})
-
-"""
-
-proc traverseCode(c: var GeneratedCode) =
-  const AllowedInToplevelC = {SummonConst, SummonGlobal, SummonThreadLocal,
-                              ProcDecl, ForeignDecl, ForeignProcDecl}
-  var i = NodePos(0)
-  while i.int < c.m.code.len:
-    let oldLen = c.code.len
-    let moveToInitSection = c.m.code[NodePos(i)].kind notin AllowedInToplevelC
-
-    gen c, c.m.code, NodePos(i)
-    next c.m.code, i
-
-    if moveToInitSection:
-      for i in oldLen ..< c.code.len:
-        c.init.add c.code[i]
-      setLen c.code, oldLen
-
-proc generateCode*(inp, outp: string) =
-  var c = initGeneratedCode(load(inp))
-
-  var co = TypeOrder()
-  traverseTypes(c.m.types, c.m.lit, co)
-
-  generateTypes(c, c.m.types, c.m.lit, co)
-  let typeDecls = move c.code
-
-  traverseCode c
-  var f = CppFile(f: open(outp, fmWrite))
-  f.write "#define NIM_INTBITS " & $c.m.intbits & "\n"
-  f.write Prelude
-  writeTokenSeq f, c.includes, c
-  writeTokenSeq f, typeDecls, c
-  writeTokenSeq f, c.data, c
-  writeTokenSeq f, c.protos, c
-  writeTokenSeq f, c.code, c
-  if c.init.len > 0:
-    f.write "void __attribute__((constructor)) init(void) {"
-    writeTokenSeq f, c.init, c
-    f.write "}\n\n"
-  f.f.close

compiler/nir/nir.nim

@@ -1,105 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Nim Intermediate Representation, designed to capture all of Nim's semantics without losing too much
-## precious information. Can easily be translated into C. And to JavaScript, hopefully.
-
-from std/os import addFileExt, `/`, createDir
-
-import std / assertions
-import ".." / [ast, modulegraphs, renderer, transf, options, msgs, lineinfos]
-import nirtypes, nirinsts, ast2ir, nirlineinfos, nirfiles, nirvm
-
-import ".." / ic / [rodfiles, bitabs]
-
-type
-  PCtx* = ref object of TPassContext
-    m: ModuleCon
-    c: ProcCon
-    oldErrorCount: int
-    bytecode: Bytecode
-
-proc newCtx*(module: PSym; g: ModuleGraph; idgen: IdGenerator): PCtx =
-  var lit = Literals()
-  var nirm = (ref NirModule)(types: initTypeGraph(lit), lit: lit)
-  var m = initModuleCon(g, g.config, idgen, module, nirm)
-  m.noModularity = true
-  PCtx(m: m, c: initProcCon(m, nil, g.config), idgen: idgen, bytecode: initBytecode(nirm))
-
-proc refresh*(c: PCtx; module: PSym; idgen: IdGenerator) =
-  #c.m = initModuleCon(c.m.graph, c.m.graph.config, idgen, module, c.m.nirm)
-  #c.m.noModularity = true
-  c.c = initProcCon(c.m, nil, c.m.graph.config)
-  c.idgen = idgen
-
-proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) =
-  if graph.repl.isNil:
-    graph.repl = newCtx(module, graph, idgen)
-    #registerAdditionalOps(PCtx graph.repl)
-  else:
-    refresh(PCtx graph.repl, module, idgen)
-
-proc setupNirReplGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
-  setupGlobalCtx(module, graph, idgen)
-  result = PCtx graph.repl
-
-proc evalStmt(c: PCtx; n: PNode) =
-  let n = transformExpr(c.m.graph, c.idgen, c.m.module, n)
-  let pc = genStmt(c.c, n)
-  #var res = ""
-  #toString c.m.nirm.code, NodePos(pc), c.m.nirm.lit.strings, c.m.nirm.lit.numbers, c.m.symnames, res
-  #res.add "\n--------------------------\n"
-  #toString res, c.m.types.g
-  if pc.int < c.m.nirm.code.len:
-    c.bytecode.interactive = c.m.graph.interactive
-    execCode c.bytecode, c.m.nirm.code, pc
-  #echo res
-
-proc runCode*(c: PPassContext; n: PNode): PNode =
-  let c = PCtx(c)
-  # don't eval errornous code:
-  if c.oldErrorCount == c.m.graph.config.errorCounter:
-    evalStmt(c, n)
-    result = newNodeI(nkEmpty, n.info)
-  else:
-    result = n
-  c.oldErrorCount = c.m.graph.config.errorCounter
-
-type
-  NirPassContext* = ref object of TPassContext
-    m: ModuleCon
-    c: ProcCon
-
-proc openNirBackend*(g: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
-  var lit = Literals()
-  var nirm = (ref NirModule)(types: initTypeGraph(lit), lit: lit)
-  let m = initModuleCon(g, g.config, idgen, module, nirm)
-  NirPassContext(m: m, c: initProcCon(m, nil, g.config), idgen: idgen)
-
-proc gen(c: NirPassContext; n: PNode) =
-  let n = transformExpr(c.m.graph, c.idgen, c.m.module, n)
-  let pc = genStmt(c.c, n)
-
-proc nirBackend*(c: PPassContext; n: PNode): PNode =
-  gen(NirPassContext(c), n)
-  result = n
-
-proc closeNirBackend*(c: PPassContext; finalNode: PNode) =
-  discard nirBackend(c, finalNode)
-
-  let c = NirPassContext(c)
-  let nimcache = getNimcacheDir(c.c.config).string
-  createDir nimcache
-  let outp = nimcache / c.m.module.name.s.addFileExt("nir")
-  #c.m.nirm.code = move c.c.code
-  try:
-    store c.m.nirm[], outp
-    echo "created: ", outp
-  except IOError:
-    rawMessage(c.c.config, errFatal, "serialization failed: " & outp)

compiler/nir/nirc.nim

@@ -1,52 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Nir Compiler.
-
-import ".." / ic / [bitabs, rodfiles]
-import nirinsts, nirtypes, nirlineinfos, nirfiles, cir
-
-proc view(filename: string) =
-  let m = load(filename)
-  var res = ""
-  allTreesToString m.code, m.lit.strings, m.lit.numbers, m.symnames, res
-  res.add "\n# TYPES\n"
-  nirtypes.toString res, m.types
-  echo res
-
-import std / [syncio, parseopt]
-
-proc writeHelp =
-  echo """Usage: nirc view|c <file.nir>"""
-  quit 0
-
-proc main =
-  var inp = ""
-  var cmd = ""
-  for kind, key, val in getopt():
-    case kind
-    of cmdArgument:
-      if cmd.len == 0: cmd = key
-      elif inp.len == 0: inp = key
-      else: quit "Error: too many arguments"
-    of cmdLongOption, cmdShortOption:
-      case key
-      of "help", "h": writeHelp()
-      of "version", "v": stdout.write "1.0\n"
-    of cmdEnd: discard
-  if inp.len == 0:
-    quit "Error: no input file specified"
-  case cmd
-  of "", "view":
-    view inp
-  of "c":
-    let outp = inp & ".c"
-    cir.generateCode inp, outp
-
-main()

compiler/nir/nirfiles.nim

@@ -1,83 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import ".." / ic / [bitabs, rodfiles]
-import nirinsts, nirtypes, nirlineinfos
-
-type
-  NirModule* = object
-    code*: Tree
-    man*: LineInfoManager
-    types*: TypeGraph
-    lit*: Literals
-    namespace*: LitId
-    intbits*: uint32
-    symnames*: SymNames
-
-proc load*(filename: string): NirModule =
-  let lit = Literals()
-  result = NirModule(lit: lit, types: initTypeGraph(lit))
-  var r = rodfiles.open(filename)
-  try:
-    r.loadHeader(nirCookie)
-    r.loadSection stringsSection
-    r.load result.lit.strings
-
-    r.loadSection numbersSection
-    r.load result.lit.numbers
-
-    r.loadSection bodiesSection
-    r.load result.code
-
-    r.loadSection typesSection
-    r.load result.types
-
-    r.loadSection sideChannelSection
-    r.load result.man
-
-    r.loadSection namespaceSection
-    r.loadPrim result.namespace
-    r.loadPrim result.intbits
-
-    r.loadSection symnamesSection
-    r.load result.symnames
-
-  finally:
-    r.close()
-
-proc store*(m: NirModule; outp: string) =
-  var r = rodfiles.create(outp)
-  try:
-    r.storeHeader(nirCookie)
-    r.storeSection stringsSection
-    r.store m.lit.strings
-
-    r.storeSection numbersSection
-    r.store m.lit.numbers
-
-    r.storeSection bodiesSection
-    r.store m.code
-
-    r.storeSection typesSection
-    r.store m.types
-
-    r.storeSection sideChannelSection
-    r.store m.man
-
-    r.storeSection namespaceSection
-    r.storePrim m.namespace
-    r.storePrim m.intbits
-
-    r.storeSection symnamesSection
-    r.store m.symnames
-
-  finally:
-    r.close()
-  if r.err != ok:
-    raise newException(IOError, "could store into: " & outp)

compiler/nir/nirinsts.nim

@@ -1,582 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## NIR instructions. Somewhat inspired by LLVM's instructions.
-
-import std / [assertions, hashes]
-import .. / ic / [bitabs, rodfiles]
-import nirlineinfos, nirtypes
-
-const
-  NirVersion = 1
-  nirCookie* = [byte(0), byte('N'), byte('I'), byte('R'),
-            byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(NirVersion)]
-
-type
-  SymId* = distinct int
-
-proc `$`*(s: SymId): string {.borrow.}
-proc hash*(s: SymId): Hash {.borrow.}
-proc `==`*(a, b: SymId): bool {.borrow.}
-
-type
-  Opcode* = enum
-    Nop,
-    ImmediateVal,
-    IntVal,
-    StrVal,
-    SymDef,
-    SymUse,
-    Typed,   # with type ID
-    PragmaId, # with Pragma ID, possible values: see PragmaKey enum
-    NilVal,
-    Label,
-    Goto,
-    CheckedGoto,
-    LoopLabel,
-    GotoLoop,  # last atom
-
-    ModuleSymUse, # `"module".x`
-
-    ArrayConstr,
-    ObjConstr,
-    Ret,
-    Yld,
-
-    Select,
-    SelectPair,  # ((values...), Label)
-    SelectList,  # (values...)
-    SelectValue, # (value)
-    SelectRange, # (valueA..valueB)
-    ForeignDecl, # Can wrap SummonGlobal, SummonThreadLocal, SummonConst
-    SummonGlobal,
-    SummonThreadLocal,
-    Summon, # x = Summon Typed <Type ID>; x begins to live
-    SummonResult,
-    SummonParam,
-    SummonConst,
-    Kill, # `Kill x`: scope end for `x`
-
-    AddrOf,
-    ArrayAt, # a[i]
-    DerefArrayAt, # a[i] where `a` is a PtrArray; `a[][i]`
-    FieldAt, # obj.field
-    DerefFieldAt, # obj[].field
-
-    Load, # a[]
-    Store, # a[] = b
-    Asgn,  # a = b
-    SetExc,
-    TestExc,
-
-    CheckedRange,
-    CheckedIndex,
-
-    Call,
-    IndirectCall,
-    CheckedCall, # call that can raise
-    CheckedIndirectCall, # call that can raise
-    CheckedAdd, # with overflow checking etc.
-    CheckedSub,
-    CheckedMul,
-    CheckedDiv,
-    CheckedMod,
-    Add,
-    Sub,
-    Mul,
-    Div,
-    Mod,
-    BitShl,
-    BitShr,
-    BitAnd,
-    BitOr,
-    BitXor,
-    BitNot,
-    BoolNot,
-    Eq,
-    Le,
-    Lt,
-    Cast,
-    NumberConv,
-    CheckedObjConv,
-    ObjConv,
-    TestOf,
-    Emit,
-    ProcDecl,
-    ForeignProcDecl,
-    PragmaPair
-
-type
-  PragmaKey* = enum
-    FastCall, StdCall, CDeclCall, SafeCall, SysCall, InlineCall, NoinlineCall, ThisCall, NoCall,
-    CoreName,
-    ExternName,
-    HeaderImport,
-    DllImport,
-    DllExport,
-    ObjExport
-
-const
-  LastAtomicValue = GotoLoop
-
-  OpcodeBits = 8'u32
-  OpcodeMask = (1'u32 shl OpcodeBits) - 1'u32
-
-  ValueProducingAtoms = {ImmediateVal, IntVal, StrVal, SymUse, NilVal}
-
-  ValueProducing* = {
-    ImmediateVal,
-    IntVal,
-    StrVal,
-    SymUse,
-    NilVal,
-    ModuleSymUse,
-    ArrayConstr,
-    ObjConstr,
-    CheckedAdd,
-    CheckedSub,
-    CheckedMul,
-    CheckedDiv,
-    CheckedMod,
-    Add,
-    Sub,
-    Mul,
-    Div,
-    Mod,
-    BitShl,
-    BitShr,
-    BitAnd,
-    BitOr,
-    BitXor,
-    BitNot,
-    BoolNot,
-    Eq,
-    Le,
-    Lt,
-    Cast,
-    NumberConv,
-    CheckedObjConv,
-    ObjConv,
-    AddrOf,
-    Load,
-    ArrayAt,
-    DerefArrayAt,
-    FieldAt,
-    DerefFieldAt,
-    TestOf
-  }
-
-type
-  Instr* = object     # 8 bytes
-    x: uint32
-    info*: PackedLineInfo
-
-template kind*(n: Instr): Opcode = Opcode(n.x and OpcodeMask)
-template operand(n: Instr): uint32 = (n.x shr OpcodeBits)
-
-template rawOperand*(n: Instr): uint32 = (n.x shr OpcodeBits)
-
-template toX(k: Opcode; operand: uint32): uint32 =
-  uint32(k) or (operand shl OpcodeBits)
-
-template toX(k: Opcode; operand: LitId): uint32 =
-  uint32(k) or (operand.uint32 shl OpcodeBits)
-
-type
-  Tree* = object
-    nodes: seq[Instr]
-
-  Values* = object
-    numbers: BiTable[int64]
-    strings: BiTable[string]
-
-type
-  PatchPos* = distinct int
-  NodePos* = distinct int
-
-const
-  InvalidPatchPos* = PatchPos(-1)
-
-proc isValid(p: PatchPos): bool {.inline.} = p.int != -1
-
-proc prepare*(tree: var Tree; info: PackedLineInfo; kind: Opcode): PatchPos =
-  result = PatchPos tree.nodes.len
-  tree.nodes.add Instr(x: toX(kind, 1'u32), info: info)
-
-proc isAtom(tree: Tree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= LastAtomicValue
-proc isAtom(tree: Tree; pos: NodePos): bool {.inline.} = tree.nodes[pos.int].kind <= LastAtomicValue
-
-proc patch*(tree: var Tree; pos: PatchPos) =
-  let pos = pos.int
-  let k = tree.nodes[pos].kind
-  assert k > LastAtomicValue
-  let distance = int32(tree.nodes.len - pos)
-  assert distance > 0
-  tree.nodes[pos].x = toX(k, cast[uint32](distance))
-
-template build*(tree: var Tree; info: PackedLineInfo; kind: Opcode; body: untyped) =
-  let pos = prepare(tree, info, kind)
-  body
-  patch(tree, pos)
-
-template buildTyped*(tree: var Tree; info: PackedLineInfo; kind: Opcode; typ: TypeId; body: untyped) =
-  let pos = prepare(tree, info, kind)
-  tree.addTyped info, typ
-  body
-  patch(tree, pos)
-
-proc len*(tree: Tree): int {.inline.} = tree.nodes.len
-
-template rawSpan(n: Instr): int = int(operand(n))
-
-proc nextChild(tree: Tree; pos: var int) {.inline.} =
-  if tree.nodes[pos].kind > LastAtomicValue:
-    assert tree.nodes[pos].operand > 0'u32
-    inc pos, tree.nodes[pos].rawSpan
-  else:
-    inc pos
-
-proc next*(tree: Tree; pos: var NodePos) {.inline.} = nextChild tree, int(pos)
-
-template firstSon*(n: NodePos): NodePos = NodePos(n.int+1)
-
-template skipTyped*(n: NodePos): NodePos = NodePos(n.int+2)
-
-iterator sons*(tree: Tree; n: NodePos): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > LastAtomicValue
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-iterator sonsFrom1*(tree: Tree; n: NodePos): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > LastAtomicValue
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  nextChild tree, pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-iterator sonsFromN*(tree: Tree; n: NodePos; toSkip = 2): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > LastAtomicValue
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  for i in 1..toSkip:
-    nextChild tree, pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-template `[]`*(t: Tree; n: NodePos): Instr = t.nodes[n.int]
-
-iterator sonsRest*(tree: Tree; parent, n: NodePos): NodePos =
-  var pos = n.int
-  assert tree[parent].kind > LastAtomicValue
-  let last = parent.int + tree[parent].rawSpan
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-proc span(tree: Tree; pos: int): int {.inline.} =
-  if tree.nodes[pos].kind <= LastAtomicValue: 1 else: int(tree.nodes[pos].operand)
-
-proc copyTree*(dest: var Tree; src: Tree) =
-  let pos = 0
-  let L = span(src, pos)
-  let d = dest.nodes.len
-  dest.nodes.setLen(d + L)
-  assert L > 0
-  for i in 0..<L:
-    dest.nodes[d+i] = src.nodes[pos+i]
-
-proc sons2*(tree: Tree; n: NodePos): (NodePos, NodePos) {.inline.} =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  result = (NodePos a, NodePos b)
-
-proc sons3*(tree: Tree; n: NodePos): (NodePos, NodePos, NodePos) {.inline.} =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  result = (NodePos a, NodePos b, NodePos c)
-
-proc sons4*(tree: Tree; n: NodePos): (NodePos, NodePos, NodePos, NodePos) {.inline.} =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  let d = c + span(tree, c)
-  result = (NodePos a, NodePos b, NodePos c, NodePos d)
-
-proc sons5*(tree: Tree; n: NodePos): (NodePos, NodePos, NodePos, NodePos, NodePos) {.inline.} =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  let d = c + span(tree, c)
-  let e = d + span(tree, d)
-  result = (NodePos a, NodePos b, NodePos c, NodePos d, NodePos e)
-
-proc typeId*(ins: Instr): TypeId {.inline.} =
-  assert ins.kind == Typed
-  result = TypeId(ins.operand)
-
-proc symId*(ins: Instr): SymId {.inline.} =
-  assert ins.kind in {SymUse, SymDef}
-  result = SymId(ins.operand)
-
-proc immediateVal*(ins: Instr): int {.inline.} =
-  assert ins.kind == ImmediateVal
-  result = cast[int](ins.operand)
-
-proc litId*(ins: Instr): LitId {.inline.} =
-  assert ins.kind in {StrVal, IntVal}
-  result = LitId(ins.operand)
-
-
-type
-  LabelId* = distinct int
-
-proc `==`*(a, b: LabelId): bool {.borrow.}
-proc hash*(a: LabelId): Hash {.borrow.}
-
-proc label*(ins: Instr): LabelId {.inline.} =
-  assert ins.kind in {Label, LoopLabel, Goto, GotoLoop, CheckedGoto}
-  result = LabelId(ins.operand)
-
-proc newLabel*(labelGen: var int): LabelId {.inline.} =
-  result = LabelId labelGen
-  inc labelGen
-
-proc newLabels*(labelGen: var int; n: int): LabelId {.inline.} =
-  result = LabelId labelGen
-  inc labelGen, n
-
-proc addNewLabel*(t: var Tree; labelGen: var int; info: PackedLineInfo; k: Opcode): LabelId =
-  assert k in {Label, LoopLabel}
-  result = LabelId labelGen
-  t.nodes.add Instr(x: toX(k, uint32(result)), info: info)
-  inc labelGen
-
-proc gotoLabel*(t: var Tree; info: PackedLineInfo; k: Opcode; L: LabelId) =
-  assert k in {Goto, GotoLoop, CheckedGoto}
-  t.nodes.add Instr(x: toX(k, uint32(L)), info: info)
-
-proc addLabel*(t: var Tree; info: PackedLineInfo; k: Opcode; L: LabelId) {.inline.} =
-  assert k in {Label, LoopLabel, Goto, GotoLoop, CheckedGoto}
-  t.nodes.add Instr(x: toX(k, uint32(L)), info: info)
-
-proc addSymUse*(t: var Tree; info: PackedLineInfo; s: SymId) {.inline.} =
-  t.nodes.add Instr(x: toX(SymUse, uint32(s)), info: info)
-
-proc addSymDef*(t: var Tree; info: PackedLineInfo; s: SymId) {.inline.} =
-  t.nodes.add Instr(x: toX(SymDef, uint32(s)), info: info)
-
-proc addNop*(t: var Tree; info: PackedLineInfo) {.inline.} =
-  t.nodes.add Instr(x: toX(Nop, 0'u32), info: info)
-
-proc addTyped*(t: var Tree; info: PackedLineInfo; typ: TypeId) {.inline.} =
-  assert typ.int >= 0
-  t.nodes.add Instr(x: toX(Typed, uint32(typ)), info: info)
-
-proc addSummon*(t: var Tree; info: PackedLineInfo; s: SymId; typ: TypeId; opc = Summon) {.inline.} =
-  assert typ.int >= 0
-  assert opc in {Summon, SummonConst, SummonGlobal, SummonThreadLocal, SummonParam, SummonResult}
-  let x = prepare(t, info, opc)
-  t.nodes.add Instr(x: toX(Typed, uint32(typ)), info: info)
-  t.nodes.add Instr(x: toX(SymDef, uint32(s)), info: info)
-  patch t, x
-
-proc addImmediateVal*(t: var Tree; info: PackedLineInfo; x: int) =
-  assert x >= 0 and x < ((1 shl 32) - OpcodeBits.int)
-  t.nodes.add Instr(x: toX(ImmediateVal, uint32(x)), info: info)
-
-proc addPragmaId*(t: var Tree; info: PackedLineInfo; x: PragmaKey) =
-  t.nodes.add Instr(x: toX(PragmaId, uint32(x)), info: info)
-
-proc addIntVal*(t: var Tree; integers: var BiTable[int64]; info: PackedLineInfo; typ: TypeId; x: int64) =
-  buildTyped t, info, NumberConv, typ:
-    t.nodes.add Instr(x: toX(IntVal, uint32(integers.getOrIncl(x))), info: info)
-
-proc boolVal*(t: var Tree; integers: var BiTable[int64]; info: PackedLineInfo; b: bool) =
-  buildTyped t, info, NumberConv, Bool8Id:
-    t.nodes.add Instr(x: toX(IntVal, uint32(integers.getOrIncl(ord b))), info: info)
-
-proc addStrVal*(t: var Tree; strings: var BiTable[string]; info: PackedLineInfo; s: string) =
-  t.nodes.add Instr(x: toX(StrVal, uint32(strings.getOrIncl(s))), info: info)
-
-proc addStrLit*(t: var Tree; info: PackedLineInfo; s: LitId) =
-  t.nodes.add Instr(x: toX(StrVal, uint32(s)), info: info)
-
-proc addNilVal*(t: var Tree; info: PackedLineInfo; typ: TypeId) =
-  buildTyped t, info, NumberConv, typ:
-    t.nodes.add Instr(x: toX(NilVal, uint32(0)), info: info)
-
-proc store*(r: var RodFile; t: Tree) = storeSeq r, t.nodes
-proc load*(r: var RodFile; t: var Tree) = loadSeq r, t.nodes
-
-proc escapeToNimLit*(s: string; result: var string) =
-  result.add '"'
-  for c in items s:
-    if c < ' ' or int(c) >= 128:
-      result.add '\\'
-      result.addInt int(c)
-    elif c == '\\':
-      result.add r"\\"
-    elif c == '\n':
-      result.add r"\n"
-    elif c == '\r':
-      result.add r"\r"
-    elif c == '\t':
-      result.add r"\t"
-    else:
-      result.add c
-  result.add '"'
-
-type
-  SymNames* = object
-    s: seq[LitId]
-
-proc `[]=`*(t: var SymNames; key: SymId; val: LitId) =
-  let k = int(key)
-  if k >= t.s.len: t.s.setLen k+1
-  t.s[k] = val
-
-proc `[]`*(t: SymNames; key: SymId): LitId =
-  let k = int(key)
-  if k < t.s.len: result = t.s[k]
-  else: result = LitId(0)
-
-template localName(s: SymId): string =
-  let name = names[s]
-  if name != LitId(0):
-    strings[name]
-  else:
-    $s.int
-
-proc store*(r: var RodFile; t: SymNames) = storeSeq(r, t.s)
-proc load*(r: var RodFile; t: var SymNames) = loadSeq(r, t.s)
-
-proc toString*(t: Tree; pos: NodePos; strings: BiTable[string]; integers: BiTable[int64];
-               names: SymNames;
-               r: var string; nesting = 0) =
-  if r.len > 0 and r[r.len-1] notin {' ', '\n', '(', '[', '{'}:
-    r.add ' '
-
-  case t[pos].kind
-  of Nop: r.add "Nop"
-  of ImmediateVal:
-    r.add $t[pos].operand
-  of IntVal:
-    r.add "IntVal "
-    r.add $integers[LitId t[pos].operand]
-  of StrVal:
-    escapeToNimLit(strings[LitId t[pos].operand], r)
-  of SymDef:
-    r.add "SymDef "
-    r.add localName(SymId t[pos].operand)
-  of SymUse:
-    r.add "SymUse "
-    r.add localName(SymId t[pos].operand)
-  of PragmaId:
-    r.add $cast[PragmaKey](t[pos].operand)
-  of Typed:
-    r.add "T<"
-    r.add $t[pos].operand
-    r.add ">"
-  of NilVal:
-    r.add "NilVal"
-  of Label:
-    # undo the nesting:
-    var spaces = r.len-1
-    while spaces >= 0 and r[spaces] == ' ': dec spaces
-    r.setLen spaces+1
-    r.add "\n  L"
-    r.add $t[pos].operand
-  of Goto, CheckedGoto, LoopLabel, GotoLoop:
-    r.add $t[pos].kind
-    r.add " L"
-    r.add $t[pos].operand
-  else:
-    r.add $t[pos].kind
-    r.add "{\n"
-    for i in 0..<(nesting+1)*2: r.add ' '
-    for p in sons(t, pos):
-      toString t, p, strings, integers, names, r, nesting+1
-    r.add "\n"
-    for i in 0..<nesting*2: r.add ' '
-    r.add "}"
-
-proc allTreesToString*(t: Tree; strings: BiTable[string]; integers: BiTable[int64];
-                       names: SymNames;
-                       r: var string) =
-  var i = 0
-  while i < t.len:
-    toString t, NodePos(i), strings, integers, names, r
-    nextChild t, i
-
-type
-  Value* = distinct Tree
-
-proc prepare*(dest: var Value; info: PackedLineInfo; k: Opcode): PatchPos {.inline.} =
-  assert k in ValueProducing - ValueProducingAtoms
-  result = prepare(Tree(dest), info, k)
-
-proc patch*(dest: var Value; pos: PatchPos) {.inline.} =
-  patch(Tree(dest), pos)
-
-proc localToValue*(info: PackedLineInfo; s: SymId): Value =
-  result = Value(Tree())
-  Tree(result).addSymUse info, s
-
-proc hasValue*(v: Value): bool {.inline.} = Tree(v).len > 0
-
-proc isEmpty*(v: Value): bool {.inline.} = Tree(v).len == 0
-
-proc extractTemp*(v: Value): SymId =
-  if hasValue(v) and Tree(v)[NodePos 0].kind == SymUse:
-    result = SymId(Tree(v)[NodePos 0].operand)
-  else:
-    result = SymId(-1)
-
-proc copyTree*(dest: var Tree; src: Value) = copyTree dest, Tree(src)
-
-proc addImmediateVal*(t: var Value; info: PackedLineInfo; x: int) =
-  assert x >= 0 and x < ((1 shl 32) - OpcodeBits.int)
-  Tree(t).nodes.add Instr(x: toX(ImmediateVal, uint32(x)), info: info)
-
-template build*(tree: var Value; info: PackedLineInfo; kind: Opcode; body: untyped) =
-  let pos = prepare(Tree(tree), info, kind)
-  body
-  patch(tree, pos)
-
-proc addTyped*(t: var Value; info: PackedLineInfo; typ: TypeId) {.inline.} =
-  addTyped(Tree(t), info, typ)
-
-template buildTyped*(tree: var Value; info: PackedLineInfo; kind: Opcode; typ: TypeId; body: untyped) =
-  let pos = prepare(tree, info, kind)
-  tree.addTyped info, typ
-  body
-  patch(tree, pos)
-
-proc addStrVal*(t: var Value; strings: var BiTable[string]; info: PackedLineInfo; s: string) =
-  addStrVal(Tree(t), strings, info, s)
-
-proc addNilVal*(t: var Value; info: PackedLineInfo; typ: TypeId) =
-  addNilVal Tree(t), info, typ
-
-proc addIntVal*(t: var Value; integers: var BiTable[int64]; info: PackedLineInfo; typ: TypeId; x: int64) =
-  addIntVal Tree(t), integers, info, typ, x

compiler/nir/nirslots.nim

@@ -1,104 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Management of slots. Similar to "register allocation"
-## in lower level languages.
-
-import std / [assertions, tables]
-import nirtypes, nirinsts
-
-type
-  SlotManagerFlag* = enum
-    ReuseTemps,
-    ReuseVars
-  SlotKind* = enum
-    Temp, Perm
-  SlotManager* = object # "register allocator"
-    live: Table[SymId, (SlotKind, TypeId)]
-    dead: Table[TypeId, seq[SymId]]
-    flags: set[SlotManagerFlag]
-    inScope: seq[SymId]
-
-proc initSlotManager*(flags: set[SlotManagerFlag]): SlotManager {.inline.} =
-  SlotManager(flags: flags)
-
-proc allocRaw(m: var SlotManager; t: TypeId; f: SlotManagerFlag; k: SlotKind;
-              symIdgen: var int32): SymId {.inline.} =
-  if f in m.flags and m.dead.hasKey(t) and m.dead[t].len > 0:
-    result = m.dead[t].pop()
-  else:
-    inc symIdgen
-    result = SymId(symIdgen)
-    m.inScope.add result
-  m.live[result] = (k, t)
-
-proc allocTemp*(m: var SlotManager; t: TypeId; symIdgen: var int32): SymId {.inline.} =
-  result = allocRaw(m, t, ReuseTemps, Temp, symIdgen)
-
-proc allocVar*(m: var SlotManager; t: TypeId; symIdgen: var int32): SymId {.inline.} =
-  result = allocRaw(m, t, ReuseVars, Perm, symIdgen)
-
-proc freeLoc*(m: var SlotManager; s: SymId) =
-  let t = m.live.getOrDefault(s)
-  assert t[1].int != 0
-  m.live.del s
-  m.dead.mgetOrPut(t[1], @[]).add s
-
-proc freeTemp*(m: var SlotManager; s: SymId) =
-  let t = m.live.getOrDefault(s)
-  if t[1].int != 0 and t[0] == Temp:
-    m.live.del s
-    m.dead.mgetOrPut(t[1], @[]).add s
-
-iterator stillAlive*(m: SlotManager): (SymId, TypeId) =
-  for k, v in pairs(m.live):
-    yield (k, v[1])
-
-proc getType*(m: SlotManager; s: SymId): TypeId {.inline.} = m.live[s][1]
-
-proc openScope*(m: var SlotManager) =
-  m.inScope.add SymId(-1) # add marker
-
-proc closeScope*(m: var SlotManager) =
-  var i = m.inScope.len - 1
-  while i >= 0:
-    if m.inScope[i] == SymId(-1):
-      m.inScope.setLen i
-      break
-    dec i
-
-when isMainModule:
-  var symIdgen: int32
-  var m = initSlotManager({ReuseTemps})
-
-  var g = initTypeGraph(Literals())
-
-  let a = g.openType ArrayTy
-  g.addBuiltinType Int8Id
-  g.addArrayLen 5
-  let finalArrayType = finishType(g, a)
-
-  let obj = g.openType ObjectDecl
-  g.addName "MyType"
-
-  g.addField "p", finalArrayType, 0
-  let objB = finishType(g, obj)
-
-  let x = m.allocTemp(objB, symIdgen)
-  assert x.int == 0
-
-  let y = m.allocTemp(objB, symIdgen)
-  assert y.int == 1
-
-  let z = m.allocTemp(Int8Id, symIdgen)
-  assert z.int == 2
-
-  m.freeLoc y
-  let y2 = m.allocTemp(objB, symIdgen)
-  assert y2.int == 1

compiler/nir/nirtypes.nim

@@ -1,475 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Type system for NIR. Close to C's type system but without its quirks.
-
-import std / [assertions, hashes]
-import .. / ic / [bitabs, rodfiles]
-
-type
-  NirTypeKind* = enum
-    VoidTy, IntTy, UIntTy, FloatTy, BoolTy, CharTy, NameVal,
-    IntVal, SizeVal, AlignVal, OffsetVal,
-    AnnotationVal,
-    ObjectTy,
-    UnionTy,
-    VarargsTy, # the `...` in a C prototype; also the last "atom"
-    APtrTy, # pointer to aliasable memory
-    UPtrTy, # pointer to unique/unaliasable memory
-    AArrayPtrTy, # pointer to array of aliasable memory
-    UArrayPtrTy, # pointer to array of unique/unaliasable memory
-    ArrayTy,
-    LastArrayTy, # array of unspecified size as a last field inside an object
-    ProcTy,
-    ObjectDecl,
-    UnionDecl,
-    FieldDecl
-
-const
-  TypeKindBits = 8'u32
-  TypeKindMask = (1'u32 shl TypeKindBits) - 1'u32
-
-type
-  TypeNode* = object     # 4 bytes
-    x: uint32
-
-template kind*(n: TypeNode): NirTypeKind = NirTypeKind(n.x and TypeKindMask)
-template operand(n: TypeNode): uint32 = (n.x shr TypeKindBits)
-
-proc integralBits*(n: TypeNode): int {.inline.} =
-  # Number of bits in the IntTy, etc. Only valid for integral types.
-  assert n.kind in {IntTy, UIntTy, FloatTy, BoolTy, CharTy}
-  result = int(n.operand)
-
-template toX(k: NirTypeKind; operand: uint32): uint32 =
-  uint32(k) or (operand shl TypeKindBits)
-
-template toX(k: NirTypeKind; operand: LitId): uint32 =
-  uint32(k) or (operand.uint32 shl TypeKindBits)
-
-type
-  TypeId* = distinct int
-
-proc `==`*(a, b: TypeId): bool {.borrow.}
-proc hash*(a: TypeId): Hash {.borrow.}
-
-type
-  Literals* = ref object
-    strings*: BiTable[string]
-    numbers*: BiTable[int64]
-
-  TypeGraph* = object
-    nodes: seq[TypeNode]
-    lit: Literals
-
-const
-  VoidId* = TypeId 0
-  Bool8Id* = TypeId 1
-  Char8Id* = TypeId 2
-  Int8Id* = TypeId 3
-  Int16Id* = TypeId 4
-  Int32Id* = TypeId 5
-  Int64Id* = TypeId 6
-  UInt8Id* = TypeId 7
-  UInt16Id* = TypeId 8
-  UInt32Id* = TypeId 9
-  UInt64Id* = TypeId 10
-  Float32Id* = TypeId 11
-  Float64Id* = TypeId 12
-  VoidPtrId* = TypeId 13
-  LastBuiltinId* = 13
-
-proc initTypeGraph*(lit: Literals): TypeGraph =
-  result = TypeGraph(nodes: @[
-    TypeNode(x: toX(VoidTy, 0'u32)),
-    TypeNode(x: toX(BoolTy, 8'u32)),
-    TypeNode(x: toX(CharTy, 8'u32)),
-    TypeNode(x: toX(IntTy, 8'u32)),
-    TypeNode(x: toX(IntTy, 16'u32)),
-    TypeNode(x: toX(IntTy, 32'u32)),
-    TypeNode(x: toX(IntTy, 64'u32)),
-    TypeNode(x: toX(UIntTy, 8'u32)),
-    TypeNode(x: toX(UIntTy, 16'u32)),
-    TypeNode(x: toX(UIntTy, 32'u32)),
-    TypeNode(x: toX(UIntTy, 64'u32)),
-    TypeNode(x: toX(FloatTy, 32'u32)),
-    TypeNode(x: toX(FloatTy, 64'u32)),
-    TypeNode(x: toX(APtrTy, 2'u32)),
-    TypeNode(x: toX(VoidTy, 0'u32))
-  ], lit: lit)
-  assert result.nodes.len == LastBuiltinId+2
-
-type
-  TypePatchPos* = distinct int
-
-const
-  InvalidTypePatchPos* = TypePatchPos(-1)
-  LastAtomicValue = VarargsTy
-
-proc isValid(p: TypePatchPos): bool {.inline.} = p.int != -1
-
-proc prepare(tree: var TypeGraph; kind: NirTypeKind): TypePatchPos =
-  result = TypePatchPos tree.nodes.len
-  tree.nodes.add TypeNode(x: toX(kind, 1'u32))
-
-proc isAtom(tree: TypeGraph; pos: int): bool {.inline.} = tree.nodes[pos].kind <= LastAtomicValue
-proc isAtom(tree: TypeGraph; pos: TypeId): bool {.inline.} = tree.nodes[pos.int].kind <= LastAtomicValue
-
-proc patch(tree: var TypeGraph; pos: TypePatchPos) =
-  let pos = pos.int
-  let k = tree.nodes[pos].kind
-  assert k > LastAtomicValue
-  let distance = int32(tree.nodes.len - pos)
-  assert distance > 0
-  tree.nodes[pos].x = toX(k, cast[uint32](distance))
-
-proc len*(tree: TypeGraph): int {.inline.} = tree.nodes.len
-
-template rawSpan(n: TypeNode): int = int(operand(n))
-
-proc nextChild(tree: TypeGraph; pos: var int) {.inline.} =
-  if tree.nodes[pos].kind > LastAtomicValue:
-    assert tree.nodes[pos].operand > 0'u32
-    inc pos, tree.nodes[pos].rawSpan
-  else:
-    inc pos
-
-iterator sons*(tree: TypeGraph; n: TypeId): TypeId =
-  var pos = n.int
-  assert tree.nodes[pos].kind > LastAtomicValue
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  while pos < last:
-    yield TypeId pos
-    nextChild tree, pos
-
-template `[]`*(t: TypeGraph; n: TypeId): TypeNode = t.nodes[n.int]
-
-proc elementType*(tree: TypeGraph; n: TypeId): TypeId {.inline.} =
-  assert tree[n].kind in {APtrTy, UPtrTy, AArrayPtrTy, UArrayPtrTy, ArrayTy, LastArrayTy}
-  result = TypeId(n.int+1)
-
-proc litId*(n: TypeNode): LitId {.inline.} =
-  assert n.kind in {NameVal, IntVal, SizeVal, AlignVal, OffsetVal, AnnotationVal, ObjectTy, UnionTy}
-  result = LitId(n.operand)
-
-proc kind*(tree: TypeGraph; n: TypeId): NirTypeKind {.inline.} = tree[n].kind
-
-proc span(tree: TypeGraph; pos: int): int {.inline.} =
-  if tree.nodes[pos].kind <= LastAtomicValue: 1 else: int(tree.nodes[pos].operand)
-
-proc sons2(tree: TypeGraph; n: TypeId): (TypeId, TypeId) =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  result = (TypeId a, TypeId b)
-
-proc sons3(tree: TypeGraph; n: TypeId): (TypeId, TypeId, TypeId) =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  result = (TypeId a, TypeId b, TypeId c)
-
-proc arrayName*(tree: TypeGraph; n: TypeId): TypeId {.inline.} =
-  assert tree[n].kind == ArrayTy
-  let (_, _, c) = sons3(tree, n)
-  result = c
-
-proc arrayLen*(tree: TypeGraph; n: TypeId): BiggestInt =
-  assert tree[n].kind == ArrayTy
-  let (_, b) = sons2(tree, n)
-  result = tree.lit.numbers[LitId tree[b].operand]
-
-proc returnType*(tree: TypeGraph; n: TypeId): (TypeId, TypeId) =
-  # Returns the positions of the return type + calling convention.
-  var pos = n.int
-  assert tree.nodes[pos].kind == ProcTy
-  let a = n.int+1
-  let b = a + span(tree, a)
-  result = (TypeId b, TypeId a) # not a typo, order is reversed
-
-iterator params*(tree: TypeGraph; n: TypeId): TypeId =
-  var pos = n.int
-  assert tree.nodes[pos].kind == ProcTy
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  nextChild tree, pos
-  nextChild tree, pos
-  while pos < last:
-    yield TypeId pos
-    nextChild tree, pos
-
-proc openType*(tree: var TypeGraph; kind: NirTypeKind): TypePatchPos =
-  assert kind in {APtrTy, UPtrTy, AArrayPtrTy, UArrayPtrTy,
-    ArrayTy, LastArrayTy, ProcTy, ObjectDecl, UnionDecl,
-    FieldDecl}
-  result = prepare(tree, kind)
-
-template typeInvariant(p: TypePatchPos) =
-  when false:
-    if tree[TypeId(p)].kind == FieldDecl:
-      var k = 0
-      for ch in sons(tree, TypeId(p)):
-        inc k
-      assert k > 2, "damn! " & $k
-
-proc sealType*(tree: var TypeGraph; p: TypePatchPos) =
-  patch tree, p
-  typeInvariant(p)
-
-proc finishType*(tree: var TypeGraph; p: TypePatchPos): TypeId =
-  # Search for an existing instance of this type in
-  # order to reduce memory consumption:
-  patch tree, p
-  typeInvariant(p)
-
-  let s = span(tree, p.int)
-  var i = 0
-  while i < p.int:
-    if tree.nodes[i].x == tree.nodes[p.int].x:
-      var isMatch = true
-      for j in 1..<s:
-        if tree.nodes[j+i].x == tree.nodes[j+p.int].x:
-          discard "still a match"
-        else:
-          isMatch = false
-          break
-      if isMatch:
-        if p.int+s == tree.len:
-          setLen tree.nodes, p.int
-        return TypeId(i)
-    nextChild tree, i
-  result = TypeId(p)
-
-proc nominalType*(tree: var TypeGraph; kind: NirTypeKind; name: string): TypeId =
-  assert kind in {ObjectTy, UnionTy}
-  let content = TypeNode(x: toX(kind, tree.lit.strings.getOrIncl(name)))
-  for i in 0..<tree.len:
-    if tree.nodes[i].x == content.x:
-      return TypeId(i)
-  result = TypeId tree.nodes.len
-  tree.nodes.add content
-
-proc addNominalType*(tree: var TypeGraph; kind: NirTypeKind; name: string) =
-  assert kind in {ObjectTy, UnionTy}
-  tree.nodes.add TypeNode(x: toX(kind, tree.lit.strings.getOrIncl(name)))
-
-proc getTypeTag*(tree: TypeGraph; t: TypeId): string =
-  assert tree[t].kind in {ObjectTy, UnionTy}
-  result = tree.lit.strings[LitId tree[t].operand]
-
-proc addVarargs*(tree: var TypeGraph) =
-  tree.nodes.add TypeNode(x: toX(VarargsTy, 0'u32))
-
-proc getFloat128Type*(tree: var TypeGraph): TypeId =
-  result = TypeId tree.nodes.len
-  tree.nodes.add TypeNode(x: toX(FloatTy, 128'u32))
-
-proc addBuiltinType*(g: var TypeGraph; id: TypeId) =
-  g.nodes.add g[id]
-
-template firstSon*(n: TypeId): TypeId = TypeId(n.int+1)
-
-proc addType*(g: var TypeGraph; t: TypeId) =
-  # We cannot simply copy `*Decl` nodes. We have to introduce `*Ty` nodes instead:
-  if g[t].kind in {ObjectDecl, UnionDecl}:
-    assert g[t.firstSon].kind == NameVal
-    let name = LitId g[t.firstSon].operand
-    if g[t].kind == ObjectDecl:
-      g.nodes.add TypeNode(x: toX(ObjectTy, name))
-    else:
-      g.nodes.add TypeNode(x: toX(UnionTy, name))
-  else:
-    let pos = t.int
-    let L = span(g, pos)
-    let d = g.nodes.len
-    g.nodes.setLen(d + L)
-    assert L > 0
-    for i in 0..<L:
-      g.nodes[d+i] = g.nodes[pos+i]
-
-proc addArrayLen*(g: var TypeGraph; len: int64) =
-  g.nodes.add TypeNode(x: toX(IntVal, g.lit.numbers.getOrIncl(len)))
-
-proc addSize*(g: var TypeGraph; s: int64) =
-  g.nodes.add TypeNode(x: toX(SizeVal, g.lit.numbers.getOrIncl(s)))
-
-proc addOffset*(g: var TypeGraph; offset: int64) =
-  g.nodes.add TypeNode(x: toX(OffsetVal, g.lit.numbers.getOrIncl(offset)))
-
-proc addAlign*(g: var TypeGraph; a: int64) =
-  g.nodes.add TypeNode(x: toX(AlignVal, g.lit.numbers.getOrIncl(a)))
-
-proc addName*(g: var TypeGraph; name: string) =
-  g.nodes.add TypeNode(x: toX(NameVal, g.lit.strings.getOrIncl(name)))
-
-proc addAnnotation*(g: var TypeGraph; name: string) =
-  g.nodes.add TypeNode(x: toX(NameVal, g.lit.strings.getOrIncl(name)))
-
-proc addField*(g: var TypeGraph; name: string; typ: TypeId; offset: int64) =
-  let f = g.openType FieldDecl
-  g.addType typ
-  g.addOffset offset
-  g.addName name
-  sealType(g, f)
-
-proc ptrTypeOf*(g: var TypeGraph; t: TypeId): TypeId =
-  let f = g.openType APtrTy
-  g.addType t
-  result = finishType(g, f)
-
-proc arrayPtrTypeOf*(g: var TypeGraph; t: TypeId): TypeId =
-  let f = g.openType AArrayPtrTy
-  g.addType t
-  result = finishType(g, f)
-
-proc store*(r: var RodFile; g: TypeGraph) =
-  storeSeq r, g.nodes
-
-proc load*(r: var RodFile; g: var TypeGraph) =
-  loadSeq r, g.nodes
-
-proc toString*(dest: var string; g: TypeGraph; i: TypeId) =
-  case g[i].kind
-  of VoidTy: dest.add "void"
-  of IntTy:
-    dest.add "i"
-    dest.addInt g[i].operand
-  of UIntTy:
-    dest.add "u"
-    dest.addInt g[i].operand
-  of FloatTy:
-    dest.add "f"
-    dest.addInt g[i].operand
-  of BoolTy:
-    dest.add "b"
-    dest.addInt g[i].operand
-  of CharTy:
-    dest.add "c"
-    dest.addInt g[i].operand
-  of NameVal, AnnotationVal:
-    dest.add g.lit.strings[LitId g[i].operand]
-  of IntVal, SizeVal, AlignVal, OffsetVal:
-    dest.add $g[i].kind
-    dest.add ' '
-    dest.add $g.lit.numbers[LitId g[i].operand]
-  of VarargsTy:
-    dest.add "..."
-  of APtrTy:
-    dest.add "aptr["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of UPtrTy:
-    dest.add "uptr["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of AArrayPtrTy:
-    dest.add "aArrayPtr["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of UArrayPtrTy:
-    dest.add "uArrayPtr["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of ArrayTy:
-    dest.add "Array["
-    let (elems, len, name) = g.sons3(i)
-    toString(dest, g, elems)
-    dest.add ", "
-    toString(dest, g, len)
-    dest.add ", "
-    toString(dest, g, name)
-    dest.add "]"
-  of LastArrayTy:
-    # array of unspecified size as a last field inside an object
-    dest.add "LastArrayTy["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of ObjectTy:
-    dest.add "object "
-    dest.add g.lit.strings[LitId g[i].operand]
-  of UnionTy:
-    dest.add "union "
-    dest.add g.lit.strings[LitId g[i].operand]
-  of ProcTy:
-    dest.add "proc["
-    for t in sons(g, i):
-      dest.add ' '
-      toString(dest, g, t)
-    dest.add "]"
-  of ObjectDecl:
-    dest.add "object["
-    for t in sons(g, i):
-      toString(dest, g, t)
-      dest.add '\n'
-    dest.add "]"
-  of UnionDecl:
-    dest.add "union["
-    for t in sons(g, i):
-      toString(dest, g, t)
-      dest.add '\n'
-    dest.add "]"
-  of FieldDecl:
-    dest.add "field["
-    for t in sons(g, i):
-      toString(dest, g, t)
-      dest.add ' '
-    dest.add "]"
-
-    when false:
-      let (typ, offset, name) = g.sons3(i)
-      toString(dest, g, typ)
-      dest.add ' '
-      toString(dest, g, offset)
-      dest.add ' '
-      toString(dest, g, name)
-
-proc toString*(dest: var string; g: TypeGraph) =
-  var i = 0
-  while i < g.len:
-    dest.add "T<"
-    dest.addInt i
-    dest.add "> "
-    toString(dest, g, TypeId i)
-    dest.add '\n'
-    nextChild g, i
-
-iterator allTypes*(g: TypeGraph; start = 0): TypeId =
-  var i = start
-  while i < g.len:
-    yield TypeId i
-    nextChild g, i
-
-iterator allTypesIncludingInner*(g: TypeGraph; start = 0): TypeId =
-  var i = start
-  while i < g.len:
-    yield TypeId i
-    inc i
-
-proc `$`(g: TypeGraph): string =
-  result = ""
-  toString(result, g)
-
-when isMainModule:
-  var g = initTypeGraph(Literals())
-
-  let a = g.openType ArrayTy
-  g.addBuiltinType Int8Id
-  g.addArrayLen 5
-  g.addName "SomeArray"
-  let finalArrayType = finishType(g, a)
-
-  let obj = g.openType ObjectDecl
-  g.nodes.add TypeNode(x: toX(NameVal, g.lit.strings.getOrIncl("MyType")))
-
-  g.addField "p", finalArrayType, 0
-  sealType(g, obj)
-
-  echo g

compiler/nir/stringcases.nim

@@ -1,200 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## included from ast2ir.nim
-
-#[
-
-case s
-of "abc", "abbd":
-  echo 1
-of "hah":
-  echo 2
-of "gah":
-  echo 3
-
-# we produce code like this:
-
-if s[0] <= 'a':
-  if s == "abc: goto L1
-  elif s == "abbd": goto L1
-else:
-  if s[2] <= 'h':
-    if s == "hah": goto L2
-    elif s == "gah": goto L3
-goto afterCase
-
-L1:
-  echo 1
-  goto afterCase
-L2:
-  echo 2
-  goto afterCase
-L3:
-  echo 3
-  goto afterCase
-
-afterCase: ...
-
-]#
-
-# We split the set of strings into 2 sets of roughly the same size.
-# The condition used for splitting is a (position, char) tuple.
-# Every string of length > position for which s[position] <= char is in one
-# set else it is in the other set.
-
-from std/sequtils import addUnique
-
-type
-  Key = (LitId, LabelId)
-
-proc splitValue(strings: BiTable[string]; a: openArray[Key]; position: int): (char, float) =
-  var cand: seq[char] = @[]
-  for t in items a:
-    let s = strings[t[0]]
-    if s.len > position: cand.addUnique s[position]
-
-  result = ('\0', -1.0)
-  for disc in items cand:
-    var hits = 0
-    for t in items a:
-      let s = strings[t[0]]
-      if s.len > position and s[position] <= disc:
-        inc hits
-    # the split is the better, the more `hits` is close to `a.len / 2`:
-    let grade = 100000.0 - abs(hits.float - a.len.float / 2.0)
-    if grade > result[1]:
-      result = (disc, grade)
-
-proc tryAllPositions(strings: BiTable[string]; a: openArray[Key]): (char, int) =
-  var m = 0
-  for t in items a:
-    m = max(m, strings[t[0]].len)
-
-  result = ('\0', -1)
-  var best = -1.0
-  for i in 0 ..< m:
-    let current = splitValue(strings, a, i)
-    if current[1] > best:
-      best = current[1]
-      result = (current[0], i)
-
-type
-  SearchKind = enum
-    LinearSearch, SplitSearch
-  SearchResult* = object
-    case kind: SearchKind
-    of LinearSearch:
-      a: seq[Key]
-    of SplitSearch:
-      span: int
-      best: (char, int)
-
-proc emitLinearSearch(strings: BiTable[string]; a: openArray[Key]; dest: var seq[SearchResult]) =
-  var d = SearchResult(kind: LinearSearch, a: @[])
-  for x in a: d.a.add x
-  dest.add d
-
-proc split(strings: BiTable[string]; a: openArray[Key]; dest: var seq[SearchResult]) =
-  if a.len <= 4:
-    emitLinearSearch strings, a, dest
-  else:
-    let best = tryAllPositions(strings, a)
-    var groupA: seq[Key] = @[]
-    var groupB: seq[Key] = @[]
-    for t in items a:
-      let s = strings[t[0]]
-      if s.len > best[1] and s[best[1]] <= best[0]:
-        groupA.add t
-      else:
-        groupB.add t
-    if groupA.len == 0 or groupB.len == 0:
-      emitLinearSearch strings, a, dest
-    else:
-      let toPatch = dest.len
-      dest.add SearchResult(kind: SplitSearch, span: 1, best: best)
-      split strings, groupA, dest
-      split strings, groupB, dest
-      let dist = dest.len - toPatch
-      assert dist > 0
-      dest[toPatch].span = dist
-
-proc toProblemDescription(c: var ProcCon; n: PNode): (seq[Key], LabelId) =
-  result = (@[], newLabels(c.labelGen, n.len))
-  assert n.kind == nkCaseStmt
-  for i in 1..<n.len:
-    let it = n[i]
-    let thisBranch = LabelId(result[1].int + i - 1)
-    if it.kind == nkOfBranch:
-      for j in 0..<it.len-1:
-        assert it[j].kind in {nkStrLit..nkTripleStrLit}
-        result[0].add (c.lit.strings.getOrIncl(it[j].strVal), thisBranch)
-
-proc decodeSolution(c: var ProcCon; dest: var Tree; s: seq[SearchResult]; i: int;
-                    selector: Value; info: PackedLineInfo) =
-  case s[i].kind
-  of SplitSearch:
-    let thenA = i+1
-    let elseA = thenA + (if s[thenA].kind == LinearSearch: 1 else: s[thenA].span)
-    let best = s[i].best
-
-    let tmp = getTemp(c, Bool8Id, info)
-    buildTyped dest, info, Asgn, Bool8Id:
-      dest.copyTree tmp
-      buildTyped dest, info, Call, Bool8Id:
-        c.addUseCodegenProc dest, "nimStrAtLe", info
-        dest.copyTree selector
-        dest.addIntVal c.lit.numbers, info, c.m.nativeIntId, best[1]
-        dest.addIntVal c.lit.numbers, info, Char8Id, best[0].int
-
-    template then() =
-      c.decodeSolution dest, s, thenA, selector, info
-    template otherwise() =
-      c.decodeSolution dest, s, elseA, selector, info
-    buildIfThenElse tmp, then, otherwise
-    freeTemp c, tmp
-
-  of LinearSearch:
-    let tmp = getTemp(c, Bool8Id, info)
-    for x in s[i].a:
-      buildTyped dest, info, Asgn, Bool8Id:
-        dest.copyTree tmp
-        buildTyped dest, info, Call, Bool8Id:
-          c.addUseCodegenProc dest, "eqStrings", info
-          dest.copyTree selector
-          dest.addStrLit info, x[0]
-      buildIf tmp:
-        c.code.gotoLabel info, Goto, x[1]
-    freeTemp c, tmp
-
-proc genStringCase(c: var ProcCon; n: PNode; d: var Value) =
-  let (problem, firstBranch) = toProblemDescription(c, n)
-  var solution: seq[SearchResult] = @[]
-  split c.lit.strings, problem, solution
-
-  # XXX Todo move complex case selector into a temporary.
-  let selector = c.genx(n[0])
-
-  let info = toLineInfo(c, n.info)
-  decodeSolution c, c.code, solution, 0, selector, info
-
-  let lend = newLabel(c.labelGen)
-  c.code.addLabel info, Goto, lend
-  for i in 1..<n.len:
-    let it = n[i]
-    let thisBranch = LabelId(firstBranch.int + i - 1)
-    c.code.addLabel info, Label, thisBranch
-    if it.kind == nkOfBranch:
-      gen(c, it.lastSon, d)
-      c.code.addLabel info, Goto, lend
-    else:
-      gen(c, it.lastSon, d)
-
-  c.code.addLabel info, Label, lend
-  freeTemp c, selector

compiler/nir/types2ir.nim

@@ -1,525 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import std / [assertions, tables, sets]
-import ".." / [ast, types, options, sighashes, modulegraphs]
-import nirtypes
-
-type
-  TypesCon* = object
-    processed: Table[ItemId, TypeId]
-    processedByName: Table[string, TypeId]
-    recursionCheck: HashSet[ItemId]
-    conf: ConfigRef
-    stringType: TypeId
-
-proc initTypesCon*(conf: ConfigRef): TypesCon =
-  TypesCon(conf: conf, stringType: TypeId(-1))
-
-proc mangle(c: var TypesCon; t: PType): string =
-  result = $sighashes.hashType(t, c.conf)
-
-template cached(c: var TypesCon; t: PType; body: untyped) =
-  result = c.processed.getOrDefault(t.itemId)
-  if result.int == 0:
-    body
-    c.processed[t.itemId] = result
-
-template cachedByName(c: var TypesCon; t: PType; body: untyped) =
-  let key = mangle(c, t)
-  result = c.processedByName.getOrDefault(key)
-  if result.int == 0:
-    body
-    c.processedByName[key] = result
-
-proc typeToIr*(c: var TypesCon; g: var TypeGraph; t: PType): TypeId
-
-proc collectFieldTypes(c: var TypesCon; g: var TypeGraph; n: PNode; dest: var Table[ItemId, TypeId]) =
-  case n.kind
-  of nkRecList:
-    for i in 0..<n.len:
-      collectFieldTypes(c, g, n[i], dest)
-  of nkRecCase:
-    assert(n[0].kind == nkSym)
-    collectFieldTypes(c, g, n[0], dest)
-    for i in 1..<n.len:
-      case n[i].kind
-      of nkOfBranch, nkElse:
-        collectFieldTypes c, g, lastSon(n[i]), dest
-      else: discard
-  of nkSym:
-    dest[n.sym.itemId] = typeToIr(c, g, n.sym.typ)
-  else:
-    assert false, "unknown node kind: " & $n.kind
-
-proc objectToIr(c: var TypesCon; g: var TypeGraph; n: PNode; fieldTypes: Table[ItemId, TypeId]; unionId: var int) =
-  case n.kind
-  of nkRecList:
-    for i in 0..<n.len:
-      objectToIr(c, g, n[i], fieldTypes, unionId)
-  of nkRecCase:
-    assert(n[0].kind == nkSym)
-    objectToIr(c, g, n[0], fieldTypes, unionId)
-    let u = openType(g, UnionDecl)
-    g.addName "u_" & $unionId
-    inc unionId
-    for i in 1..<n.len:
-      case n[i].kind
-      of nkOfBranch, nkElse:
-        let subObj = openType(g, ObjectDecl)
-        g.addName "uo_" & $unionId & "_" & $i
-        objectToIr c, g, lastSon(n[i]), fieldTypes, unionId
-        sealType(g, subObj)
-      else: discard
-    sealType(g, u)
-  of nkSym:
-    g.addField n.sym.name.s & "_" & $n.sym.position, fieldTypes[n.sym.itemId], n.sym.offset
-  else:
-    assert false, "unknown node kind: " & $n.kind
-
-proc objectToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  if t.baseClass != nil:
-    # ensure we emitted the base type:
-    discard typeToIr(c, g, t.baseClass)
-
-  var unionId = 0
-  var fieldTypes = initTable[ItemId, TypeId]()
-  collectFieldTypes c, g, t.n, fieldTypes
-  let obj = openType(g, ObjectDecl)
-  g.addName mangle(c, t)
-  g.addSize c.conf.getSize(t)
-  g.addAlign c.conf.getAlign(t)
-
-  if t.baseClass != nil:
-    g.addNominalType(ObjectTy, mangle(c, t.baseClass))
-  else:
-    g.addBuiltinType VoidId # object does not inherit
-    if not lacksMTypeField(t):
-      let f2 = g.openType FieldDecl
-      let voidPtr = openType(g, APtrTy)
-      g.addBuiltinType(VoidId)
-      sealType(g, voidPtr)
-      g.addOffset 0 # type field is always at offset 0
-      g.addName "m_type"
-      sealType(g, f2) # FieldDecl
-
-  objectToIr c, g, t.n, fieldTypes, unionId
-  result = finishType(g, obj)
-
-proc objectHeaderToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  result = g.nominalType(ObjectTy, mangle(c, t))
-
-proc tupleToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  var fieldTypes = newSeq[TypeId](t.len)
-  for i in 0..<t.len:
-    fieldTypes[i] = typeToIr(c, g, t[i])
-  let obj = openType(g, ObjectDecl)
-  g.addName mangle(c, t)
-  g.addSize c.conf.getSize(t)
-  g.addAlign c.conf.getAlign(t)
-
-  var accum = OffsetAccum(maxAlign: 1)
-  for i in 0..<t.len:
-    let child = t[i]
-    g.addField "f_" & $i, fieldTypes[i], accum.offset
-
-    computeSizeAlign(c.conf, child)
-    accum.align(child.align)
-    accum.inc(int32(child.size))
-  result = finishType(g, obj)
-
-proc procToIr(c: var TypesCon; g: var TypeGraph; t: PType; addEnv = false): TypeId =
-  var fieldTypes = newSeq[TypeId](0)
-  for i in 0..<t.len:
-    if t[i] == nil or not isCompileTimeOnly(t[i]):
-      fieldTypes.add typeToIr(c, g, t[i])
-  let obj = openType(g, ProcTy)
-
-  case t.callConv
-  of ccNimCall, ccFastCall, ccClosure: g.addAnnotation "__fastcall"
-  of ccStdCall: g.addAnnotation "__stdcall"
-  of ccCDecl: g.addAnnotation "__cdecl"
-  of ccSafeCall: g.addAnnotation "__safecall"
-  of ccSysCall: g.addAnnotation "__syscall"
-  of ccInline: g.addAnnotation "__inline"
-  of ccNoInline: g.addAnnotation "__noinline"
-  of ccThisCall: g.addAnnotation "__thiscall"
-  of ccNoConvention, ccMember: g.addAnnotation ""
-
-  for i in 0..<fieldTypes.len:
-    g.addType fieldTypes[i]
-
-  if addEnv:
-    let a = openType(g, APtrTy)
-    g.addBuiltinType(VoidId)
-    sealType(g, a)
-
-  if tfVarargs in t.flags:
-    g.addVarargs()
-  result = finishType(g, obj)
-
-proc nativeInt(c: TypesCon): TypeId =
-  case c.conf.target.intSize
-  of 2: result = Int16Id
-  of 4: result = Int32Id
-  else: result = Int64Id
-
-proc openArrayPayloadType*(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  let e = elementType(t)
-  let elementType = typeToIr(c, g, e)
-  let arr = g.openType AArrayPtrTy
-  g.addType elementType
-  result = finishType(g, arr) # LastArrayTy
-
-proc openArrayToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  # object (a: ArrayPtr[T], len: int)
-  let e = elementType(t)
-  let mangledBase = mangle(c, e)
-  let typeName = "NimOpenArray" & mangledBase
-
-  let elementType = typeToIr(c, g, e)
-  #assert elementType.int >= 0, typeToString(t)
-
-  let p = openType(g, ObjectDecl)
-  g.addName typeName
-  g.addSize c.conf.target.ptrSize*2
-  g.addAlign c.conf.target.ptrSize
-
-  let f = g.openType FieldDecl
-  let arr = g.openType AArrayPtrTy
-  g.addType elementType
-  sealType(g, arr) # LastArrayTy
-  g.addOffset 0
-  g.addName "data"
-  sealType(g, f) # FieldDecl
-
-  g.addField "len", c.nativeInt, c.conf.target.ptrSize
-
-  result = finishType(g, p) # ObjectDecl
-
-proc strPayloadType(c: var TypesCon; g: var TypeGraph): (string, TypeId) =
-  result = ("NimStrPayload", TypeId(-1))
-  let p = openType(g, ObjectDecl)
-  g.addName result[0]
-  g.addSize c.conf.target.ptrSize*2
-  g.addAlign c.conf.target.ptrSize
-
-  g.addField "cap", c.nativeInt, 0
-
-  let f = g.openType FieldDecl
-  let arr = g.openType LastArrayTy
-  g.addBuiltinType Char8Id
-  result[1] = finishType(g, arr) # LastArrayTy
-  g.addOffset c.conf.target.ptrSize # comes after the len field
-  g.addName "data"
-  sealType(g, f) # FieldDecl
-
-  sealType(g, p)
-
-proc strPayloadPtrType*(c: var TypesCon; g: var TypeGraph): (TypeId, TypeId) =
-  let (mangled, arrayType) = strPayloadType(c, g)
-  let ffp = g.openType APtrTy
-  g.addNominalType ObjectTy, mangled
-  result = (finishType(g, ffp), arrayType) # APtrTy
-
-proc stringToIr(c: var TypesCon; g: var TypeGraph): TypeId =
-  #[
-
-    NimStrPayload = object
-      cap: int
-      data: UncheckedArray[char]
-
-    NimStringV2 = object
-      len: int
-      p: ptr NimStrPayload
-
-  ]#
-  let payload = strPayloadType(c, g)
-
-  let str = openType(g, ObjectDecl)
-  g.addName "NimStringV2"
-  g.addSize c.conf.target.ptrSize*2
-  g.addAlign c.conf.target.ptrSize
-
-  g.addField "len", c.nativeInt, 0
-
-  let fp = g.openType FieldDecl
-  let ffp = g.openType APtrTy
-  g.addNominalType ObjectTy, "NimStrPayload"
-  sealType(g, ffp) # APtrTy
-  g.addOffset c.conf.target.ptrSize # comes after 'len' field
-  g.addName "p"
-  sealType(g, fp) # FieldDecl
-
-  result = finishType(g, str) # ObjectDecl
-
-proc seqPayloadType(c: var TypesCon; g: var TypeGraph; t: PType): (string, TypeId) =
-  #[
-    NimSeqPayload[T] = object
-      cap: int
-      data: UncheckedArray[T]
-  ]#
-  let e = elementType(t)
-  result = (mangle(c, e), TypeId(-1))
-  let payloadName = "NimSeqPayload" & result[0]
-
-  let elementType = typeToIr(c, g, e)
-
-  let p = openType(g, ObjectDecl)
-  g.addName payloadName
-  g.addSize c.conf.target.intSize
-  g.addAlign c.conf.target.intSize
-
-  g.addField "cap", c.nativeInt, 0
-
-  let f = g.openType FieldDecl
-  let arr = g.openType LastArrayTy
-  g.addType elementType
-  # DO NOT USE `finishType` here as it is an inner type. This is subtle and we
-  # probably need an even better API here.
-  sealType(g, arr)
-  result[1] = TypeId(arr)
-
-  g.addOffset c.conf.target.ptrSize
-  g.addName "data"
-  sealType(g, f) # FieldDecl
-
-  sealType(g, p)
-
-proc seqPayloadPtrType*(c: var TypesCon; g: var TypeGraph; t: PType): (TypeId, TypeId) =
-  let (mangledBase, arrayType) = seqPayloadType(c, g, t)
-  let ffp = g.openType APtrTy
-  g.addNominalType ObjectTy, "NimSeqPayload" & mangledBase
-  result = (finishType(g, ffp), arrayType) # APtrTy
-
-proc seqToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  #[
-    NimSeqV2*[T] = object
-      len: int
-      p: ptr NimSeqPayload[T]
-  ]#
-  let (mangledBase, _) = seqPayloadType(c, g, t)
-
-  let sq = openType(g, ObjectDecl)
-  g.addName "NimSeqV2" & mangledBase
-  g.addSize c.conf.getSize(t)
-  g.addAlign c.conf.getAlign(t)
-
-  g.addField "len", c.nativeInt, 0
-
-  let fp = g.openType FieldDecl
-  let ffp = g.openType APtrTy
-  g.addNominalType ObjectTy, "NimSeqPayload" & mangledBase
-  sealType(g, ffp) # APtrTy
-  g.addOffset c.conf.target.ptrSize
-  g.addName "p"
-  sealType(g, fp) # FieldDecl
-
-  result = finishType(g, sq) # ObjectDecl
-
-
-proc closureToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  # struct {fn(args, void* env), env}
-  # typedef struct {$n" &
-  #        "N_NIMCALL_PTR($2, ClP_0) $3;$n" &
-  #        "void* ClE_0;$n} $1;$n"
-  let mangledBase = mangle(c, t)
-  let typeName = "NimClosure" & mangledBase
-
-  let procType = procToIr(c, g, t, addEnv=true)
-
-  let p = openType(g, ObjectDecl)
-  g.addName typeName
-  g.addSize c.conf.getSize(t)
-  g.addAlign c.conf.getAlign(t)
-
-  let f = g.openType FieldDecl
-  g.addType procType
-  g.addOffset 0
-  g.addName "ClP_0"
-  sealType(g, f) # FieldDecl
-
-  let f2 = g.openType FieldDecl
-  let voidPtr = openType(g, APtrTy)
-  g.addBuiltinType(VoidId)
-  sealType(g, voidPtr)
-
-  g.addOffset c.conf.target.ptrSize
-  g.addName "ClE_0"
-  sealType(g, f2) # FieldDecl
-
-  result = finishType(g, p) # ObjectDecl
-
-proc bitsetBasetype*(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  let s = int(getSize(c.conf, t))
-  case s
-  of 1: result = UInt8Id
-  of 2: result = UInt16Id
-  of 4: result = UInt32Id
-  of 8: result = UInt64Id
-  else: result = UInt8Id
-
-proc typeToIr*(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  if t == nil: return VoidId
-  case t.kind
-  of tyInt:
-    case int(getSize(c.conf, t))
-    of 2: result = Int16Id
-    of 4: result = Int32Id
-    else: result = Int64Id
-  of tyInt8: result = Int8Id
-  of tyInt16: result = Int16Id
-  of tyInt32: result = Int32Id
-  of tyInt64: result = Int64Id
-  of tyFloat:
-    case int(getSize(c.conf, t))
-    of 4: result = Float32Id
-    else: result = Float64Id
-  of tyFloat32: result = Float32Id
-  of tyFloat64: result = Float64Id
-  of tyFloat128: result = getFloat128Type(g)
-  of tyUInt:
-    case int(getSize(c.conf, t))
-    of 2: result = UInt16Id
-    of 4: result = UInt32Id
-    else: result = UInt64Id
-  of tyUInt8: result = UInt8Id
-  of tyUInt16: result = UInt16Id
-  of tyUInt32: result = UInt32Id
-  of tyUInt64: result = UInt64Id
-  of tyBool: result = Bool8Id
-  of tyChar: result = Char8Id
-  of tyVoid: result = VoidId
-  of tySink, tyGenericInst, tyDistinct, tyAlias, tyOwned, tyRange:
-    result = typeToIr(c, g, t.skipModifier)
-  of tyEnum:
-    if firstOrd(c.conf, t) < 0:
-      result = Int32Id
-    else:
-      case int(getSize(c.conf, t))
-      of 1: result = UInt8Id
-      of 2: result = UInt16Id
-      of 4: result = Int32Id
-      of 8: result = Int64Id
-      else: result = Int32Id
-  of tyOrdinal, tyGenericBody, tyGenericParam, tyInferred, tyStatic:
-    if t.len > 0:
-      result = typeToIr(c, g, t.skipModifier)
-    else:
-      result = TypeId(-1)
-  of tyFromExpr:
-    if t.n != nil and t.n.typ != nil:
-      result = typeToIr(c, g, t.n.typ)
-    else:
-      result = TypeId(-1)
-  of tyArray:
-    cached(c, t):
-      var n = toInt64(lengthOrd(c.conf, t))
-      if n <= 0: n = 1   # make an array of at least one element
-      let elemType = typeToIr(c, g, t.elementType)
-      let a = openType(g, ArrayTy)
-      g.addType(elemType)
-      g.addArrayLen n
-      g.addName mangle(c, t)
-      result = finishType(g, a)
-  of tyPtr, tyRef:
-    cached(c, t):
-      let e = t.elementType
-      if e.kind == tyUncheckedArray:
-        let elemType = typeToIr(c, g, e.elementType)
-        let a = openType(g, AArrayPtrTy)
-        g.addType(elemType)
-        result = finishType(g, a)
-      else:
-        let elemType = typeToIr(c, g, t.elementType)
-        let a = openType(g, APtrTy)
-        g.addType(elemType)
-        result = finishType(g, a)
-  of tyVar, tyLent:
-    cached(c, t):
-      let e = t.elementType
-      if e.skipTypes(abstractInst).kind in {tyOpenArray, tyVarargs}:
-        # skip the modifier, `var openArray` is a (ptr, len) pair too:
-        result = typeToIr(c, g, e)
-      else:
-        let elemType = typeToIr(c, g, e)
-        let a = openType(g, APtrTy)
-        g.addType(elemType)
-        result = finishType(g, a)
-  of tySet:
-    let s = int(getSize(c.conf, t))
-    case s
-    of 1: result = UInt8Id
-    of 2: result = UInt16Id
-    of 4: result = UInt32Id
-    of 8: result = UInt64Id
-    else:
-      # array[U8, s]
-      cached(c, t):
-        let a = openType(g, ArrayTy)
-        g.addType(UInt8Id)
-        g.addArrayLen s
-        g.addName mangle(c, t)
-        result = finishType(g, a)
-  of tyPointer, tyNil:
-    # tyNil can happen for code like: `const CRAP = nil` which we have in posix.nim
-    let a = openType(g, APtrTy)
-    g.addBuiltinType(VoidId)
-    result = finishType(g, a)
-  of tyObject:
-    # Objects are special as they can be recursive in Nim. This is easily solvable.
-    # We check if we are already "processing" t. If so, we produce `ObjectTy`
-    # instead of `ObjectDecl`.
-    cached(c, t):
-      if not c.recursionCheck.containsOrIncl(t.itemId):
-        result = objectToIr(c, g, t)
-      else:
-        result = objectHeaderToIr(c, g, t)
-  of tyTuple:
-    cachedByName(c, t):
-      result = tupleToIr(c, g, t)
-  of tyProc:
-    cached(c, t):
-      if t.callConv == ccClosure:
-        result = closureToIr(c, g, t)
-      else:
-        result = procToIr(c, g, t)
-  of tyVarargs, tyOpenArray:
-    cached(c, t):
-      result = openArrayToIr(c, g, t)
-  of tyString:
-    if c.stringType.int < 0:
-      result = stringToIr(c, g)
-      c.stringType = result
-    else:
-      result = c.stringType
-  of tySequence:
-    cachedByName(c, t):
-      result = seqToIr(c, g, t)
-  of tyCstring:
-    cached(c, t):
-      let a = openType(g, AArrayPtrTy)
-      g.addBuiltinType Char8Id
-      result = finishType(g, a)
-  of tyUncheckedArray:
-    # We already handled the `ptr UncheckedArray` in a special way.
-    cached(c, t):
-      let elemType = typeToIr(c, g, t.elementType)
-      let a = openType(g, LastArrayTy)
-      g.addType(elemType)
-      result = finishType(g, a)
-  of tyUntyped, tyTyped:
-    # this avoids a special case for system.echo which is not a generic but
-    # uses `varargs[typed]`:
-    result = VoidId
-  of tyNone, tyEmpty, tyTypeDesc,
-     tyGenericInvocation, tyProxy, tyBuiltInTypeClass,
-     tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass,
-     tyAnd, tyOr, tyNot, tyAnything, tyConcept, tyIterable, tyForward:
-    result = TypeId(-1)

compiler/nodekinds.nim

@@ -204,6 +204,7 @@ type
     nkModuleRef           # for .rod file support: A (moduleId, itemId) pair
     nkReplayAction        # for .rod file support: A replay action
     nkNilRodNode          # for .rod file support: a 'nil' PNode
+    nkOpenSym             # container for captured sym that can be overriden by local symbols
 
 const
   nkCallKinds* = {nkCall, nkInfix, nkPrefix, nkPostfix,

compiler/options.nim

@@ -139,7 +139,6 @@ type
     backendCpp = "cpp"
     backendJs = "js"
     backendObjc = "objc"
-    backendNir = "nir"
     # backendNimscript = "nimscript" # this could actually work
     # backendLlvm = "llvm" # probably not well supported; was cmdCompileToLLVM
 
@@ -147,7 +146,6 @@ type
     cmdNone # not yet processed command
     cmdUnknown # command unmapped
     cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS,
-    cmdCompileToNir,
     cmdCrun # compile and run in nimache
     cmdTcc # run the project via TCC backend
     cmdCheck # semantic checking for whole project
@@ -176,12 +174,11 @@ type
 
 const
   cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,
-                  cmdCompileToJS, cmdCrun, cmdCompileToNir}
+                  cmdCompileToJS, cmdCrun}
   cmdDocLike* = {cmdDoc0, cmdDoc, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
                  cmdCtags, cmdBuildindex}
 
 type
-  NimVer* = tuple[major: int, minor: int, patch: int]
   TStringSeq* = seq[string]
   TGCMode* = enum             # the selected GC
     gcUnselected = "unselected"
@@ -228,8 +225,11 @@ type
     strictDefs,
     strictCaseObjects,
     inferGenericTypes,
-    genericsOpenSym,
+    openSym, # remove nfDisabledOpenSym when this is default
+    # alternative to above:
+    genericsOpenSym
     vtables
+    typeBoundOps
 
   LegacyFeature* = enum
     allowSemcheckedAstModification,
@@ -246,13 +246,15 @@ type
     emitGenerics
       ## generics are emitted in the module that contains them.
       ## Useful for libraries that rely on local passC
+    jsNoLambdaLifting
+      ## Old transformation for closures in JS backend
 
   SymbolFilesOption* = enum
     disabledSf, writeOnlySf, readOnlySf, v2Sf, stressTest
 
   TSystemCC* = enum
     ccNone, ccGcc, ccNintendoSwitch, ccLLVM_Gcc, ccCLang, ccBcc, ccVcc,
-    ccTcc, ccEnv, ccIcl, ccIcc, ccClangCl
+    ccTcc, ccEnv, ccIcl, ccIcc, ccClangCl, ccHipcc, ccNvcc
 
   ExceptionSystem* = enum
     excNone,   # no exception system selected yet
@@ -368,7 +370,6 @@ type
     arguments*: string ## the arguments to be passed to the program that
                        ## should be run
     ideCmd*: IdeCmd
-    oldNewlines*: bool
     cCompiler*: TSystemCC # the used compiler
     modifiedyNotes*: TNoteKinds # notes that have been set/unset from either cmdline/configs
     cmdlineNotes*: TNoteKinds # notes that have been set/unset from cmdline
@@ -395,8 +396,7 @@ type
     outDir*: AbsoluteDir
     jsonBuildFile*: AbsoluteFile
     prefixDir*, libpath*, nimcacheDir*: AbsoluteDir
-    nimStdlibVersion*: NimVer
-    dllOverrides, moduleOverrides*, cfileSpecificOptions*: StringTableRef
+    dllOverrides*, moduleOverrides*, cfileSpecificOptions*: StringTableRef
     projectName*: string # holds a name like 'nim'
     projectPath*: AbsoluteDir # holds a path like /home/alice/projects/nim/compiler/
     projectFull*: AbsoluteFile # projectPath/projectName
@@ -408,7 +408,6 @@ type
     commandArgs*: seq[string] # any arguments after the main command
     commandLine*: string
     extraCmds*: seq[string] # for writeJsonBuildInstructions
-    keepComments*: bool # whether the parser needs to keep comments
     implicitImports*: seq[string] # modules that are to be implicitly imported
     implicitIncludes*: seq[string] # modules that are to be implicitly included
     docSeeSrcUrl*: string # if empty, no seeSrc will be generated. \
@@ -449,16 +448,6 @@ type
     clientProcessId*: int
 
 
-proc parseNimVersion*(a: string): NimVer =
-  # could be moved somewhere reusable
-  result = default(NimVer)
-  if a.len > 0:
-    let b = a.split(".")
-    assert b.len == 3, a
-    template fn(i) = result[i] = b[i].parseInt # could be optimized if needed
-    fn(0)
-    fn(1)
-    fn(2)
 
 proc assignIfDefault*[T](result: var T, val: T, def = default(T)) =
   ## if `result` was already assigned to a value (that wasn't `def`), this is a noop.
@@ -592,7 +581,6 @@ proc newConfigRef*(): ConfigRef =
     command: "", # the main command (e.g. cc, check, scan, etc)
     commandArgs: @[], # any arguments after the main command
     commandLine: "",
-    keepComments: true, # whether the parser needs to keep comments
     implicitImports: @[], # modules that are to be implicitly imported
     implicitIncludes: @[], # modules that are to be implicitly included
     docSeeSrcUrl: "",
@@ -633,12 +621,6 @@ proc newPartialConfigRef*(): ConfigRef =
 proc cppDefine*(c: ConfigRef; define: string) =
   c.cppDefines.incl define
 
-proc getStdlibVersion*(conf: ConfigRef): NimVer =
-  if conf.nimStdlibVersion == (0,0,0):
-    let s = conf.symbols.getOrDefault("nimVersion", "")
-    conf.nimStdlibVersion = s.parseNimVersion
-  result = conf.nimStdlibVersion
-
 proc isDefined*(conf: ConfigRef; symbol: string): bool =
   if conf.symbols.hasKey(symbol):
     result = true

compiler/packages.nim

@@ -51,3 +51,11 @@ func belongsToProjectPackage*(conf: ConfigRef, sym: PSym): bool =
   ## See Also:
   ## * `modulegraphs.belongsToStdlib`
   conf.mainPackageId == sym.getPackageId
+
+func belongsToProjectPackageMaybeNil*(conf: ConfigRef, sym: PSym): bool =
+  ## Return whether the symbol belongs to the project's package.
+  ## Returns `false` if `sym` is nil.
+  ##
+  ## See Also:
+  ## * `modulegraphs.belongsToStdlib`
+  sym != nil and conf.mainPackageId == sym.getPackageId

compiler/parampatterns.nim

@@ -266,7 +266,7 @@ proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
     if skipTypes(n.typ, abstractPtrs-{tyTypeDesc}).kind in
         {tyOpenArray, tyTuple, tyObject}:
       result = isAssignable(owner, n[1])
-    elif compareTypes(n.typ, n[1].typ, dcEqIgnoreDistinct):
+    elif compareTypes(n.typ, n[1].typ, dcEqIgnoreDistinct, {IgnoreRangeShallow}):
       # types that are equal modulo distinction preserve l-value:
       result = isAssignable(owner, n[1])
   of nkHiddenDeref:
@@ -283,8 +283,15 @@ proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
   of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
     result = isAssignable(owner, n[0])
   of nkCallKinds:
-    # builtin slice keeps lvalue-ness:
-    if getMagic(n) in {mArrGet, mSlice}:
+    let m = getMagic(n)
+    if m == mSlice:
+      # builtin slice keeps l-value-ness
+      # except for pointers because slice dereferences
+      if n[1].typ.kind == tyPtr:
+        result = arLValue
+      else:
+        result = isAssignable(owner, n[1])
+    elif m == mArrGet:
       result = isAssignable(owner, n[1])
     elif n.typ != nil:
       case n.typ.kind

compiler/parser.nim

@@ -1401,7 +1401,7 @@ proc primary(p: var Parser, mode: PrimaryMode): PNode =
     result = primarySuffix(p, result, baseInd, mode)
 
 proc binaryNot(p: var Parser; a: PNode): PNode =
-  if p.tok.tokType == tkNot:
+  if p.tok.tokType == tkNot and p.tok.indent < 0:
     let notOpr = newIdentNodeP(p.tok.ident, p)
     getTok(p)
     optInd(p, notOpr)

compiler/passes.nim

@@ -42,10 +42,7 @@ proc makePass*(open: TPassOpen = nil,
                process: TPassProcess = nil,
                close: TPassClose = nil,
                isFrontend = false): TPass =
-  result.open = open
-  result.close = close
-  result.process = process
-  result.isFrontend = isFrontend
+  result = (open, process, close, isFrontend)
 
 const
   maxPasses = 10
@@ -100,8 +97,8 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
                     stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
   var
-    p: Parser
-    a: TPassContextArray
+    p: Parser = default(Parser)
+    a: TPassContextArray = default(TPassContextArray)
     s: PLLStream
     fileIdx = module.fileIdx
   prepareConfigNotes(graph, module)

compiler/pipelines.nim

@@ -13,7 +13,6 @@ when not defined(leanCompiler):
 import std/[syncio, objectdollar, assertions, tables, strutils, strtabs]
 import renderer
 import ic/replayer
-import nir/nir
 
 proc setPipeLinePass*(graph: ModuleGraph; pass: PipelinePass) =
   graph.pipelinePass = pass
@@ -45,16 +44,11 @@ proc processPipeline(graph: ModuleGraph; semNode: PNode; bModule: PPassContext):
       result = nil
   of EvalPass, InterpreterPass:
     result = interpreterCode(bModule, semNode)
-  of NirReplPass:
-    result = runCode(bModule, semNode)
-  of NirPass:
-    result = nirBackend(bModule, semNode)
   of NonePass:
     raiseAssert "use setPipeLinePass to set a proper PipelinePass"
 
-proc processImplicitImports(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind,
-                      m: PSym, ctx: PContext, bModule: PPassContext, idgen: IdGenerator,
-                      ) =
+proc processImplicitImports*(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind,
+                             m: PSym, ctx: PContext, bModule: PPassContext, idgen: IdGenerator) =
   # XXX fixme this should actually be relative to the config file!
   let relativeTo = toFullPath(graph.config, m.info)
   for module in items(implicits):
@@ -69,7 +63,7 @@ proc processImplicitImports(graph: ModuleGraph; implicits: seq[string], nodeKind
       if semNode == nil or processPipeline(graph, semNode, bModule) == nil:
         break
 
-proc prePass(c: PContext; n: PNode) =
+proc prePass*(c: PContext; n: PNode) =
   for son in n:
     if son.kind == nkPragma:
       for s in son:
@@ -111,8 +105,6 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
     case graph.pipelinePass
     of CgenPass:
       setupCgen(graph, module, idgen)
-    of NirPass:
-      openNirBackend(graph, module, idgen)
     of JSgenPass:
       when not defined(leanCompiler):
         setupJSgen(graph, module, idgen)
@@ -120,8 +112,6 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
         nil
     of EvalPass, InterpreterPass:
       setupEvalGen(graph, module, idgen)
-    of NirReplPass:
-      setupNirReplGen(graph, module, idgen)
     of GenDependPass:
       setupDependPass(graph, module, idgen)
     of Docgen2Pass:
@@ -209,10 +199,6 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
       discard finalJSCodeGen(graph, bModule, finalNode)
   of EvalPass, InterpreterPass:
     discard interpreterCode(bModule, finalNode)
-  of NirReplPass:
-    discard runCode(bModule, finalNode)
-  of NirPass:
-    closeNirBackend(bModule, finalNode)
   of SemPass, GenDependPass:
     discard
   of Docgen2Pass, Docgen2TexPass:

compiler/pragmas.nim

@@ -81,14 +81,14 @@ const
     wRequiresInit, wNoalias, wAlign, wNoInit} - {wExportNims, wNodecl} # why exclude these?
   varPragmas* = declPragmas + {wVolatile, wRegister, wThreadVar,
     wMagic, wHeader, wCompilerProc, wCore, wDynlib,
-    wNoInit, wCompileTime, wGlobal,
+    wNoInit, wCompileTime, wGlobal, wLiftLocals,
     wGensym, wInject, wCodegenDecl,
     wGuard, wGoto, wCursor, wNoalias, wAlign}
   constPragmas* = declPragmas + {wHeader, wMagic,
     wGensym, wInject,
     wIntDefine, wStrDefine, wBoolDefine, wDefine,
     wCompilerProc, wCore}
-  paramPragmas* = {wNoalias, wInject, wGensym, wByRef, wByCopy, wCodegenDecl}
+  paramPragmas* = {wNoalias, wInject, wGensym, wByRef, wByCopy, wCodegenDecl, wExportc, wExportCpp}
   letPragmas* = varPragmas
   procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNoSideEffect,
                       wThread, wRaises, wEffectsOf, wLocks, wTags, wForbids, wGcSafe,
@@ -156,16 +156,16 @@ proc pragmaEnsures(c: PContext, n: PNode) =
 proc setExternName(c: PContext; s: PSym, extname: string, info: TLineInfo) =
   # special cases to improve performance:
   if extname == "$1":
-    s.loc.r = rope(s.name.s)
+    s.loc.snippet = rope(s.name.s)
   elif '$' notin extname:
-    s.loc.r = rope(extname)
+    s.loc.snippet = rope(extname)
   else:
     try:
-      s.loc.r = rope(extname % s.name.s)
+      s.loc.snippet = rope(extname % s.name.s)
     except ValueError:
       localError(c.config, info, "invalid extern name: '" & extname & "'. (Forgot to escape '$'?)")
   when hasFFI:
-    s.cname = $s.loc.r
+    s.cname = $s.loc.snippet
 
 
 proc makeExternImport(c: PContext; s: PSym, extname: string, info: TLineInfo) =
@@ -370,7 +370,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) =
 proc processNote(c: PContext, n: PNode) =
   template handleNote(enumVals, notes) =
     let x = findStr(enumVals.a, enumVals.b, n[0][1].ident.s, errUnknown)
-    if x !=  errUnknown:
+    if x != errUnknown:
       nk = TNoteKind(x)
       let x = c.semConstBoolExpr(c, n[1])
       n[1] = x
@@ -480,6 +480,18 @@ proc processOption(c: PContext, n: PNode, resOptions: var TOptions) =
     # calling conventions (boring...):
     localError(c.config, n.info, "option expected")
 
+proc checkPushedPragma(c: PContext, n: PNode) =
+  let keyDeep = n.kind in nkPragmaCallKinds and n.len > 1
+  var key = if keyDeep: n[0] else: n
+  if key.kind in nkIdentKinds:
+    let ident = considerQuotedIdent(c, key)
+    var userPragma = strTableGet(c.userPragmas, ident)
+    if userPragma == nil:
+      let k = whichKeyword(ident)
+      # TODO: might as well make a list which is not accepted by `push`: emit, cast etc.
+      if k == wEmit:
+        localError(c.config, n.info, "an 'emit' pragma cannot be pushed")
+
 proc processPush(c: PContext, n: PNode, start: int) =
   if n[start-1].kind in nkPragmaCallKinds:
     localError(c.config, n.info, "'push' cannot have arguments")
@@ -487,6 +499,7 @@ proc processPush(c: PContext, n: PNode, start: int) =
   for i in start..<n.len:
     if not tryProcessOption(c, n[i], c.config.options):
       # simply store it somewhere:
+      checkPushedPragma(c, n[i])
       if x.otherPragmas.isNil:
         x.otherPragmas = newNodeI(nkPragma, n.info)
       x.otherPragmas.add n[i]
@@ -709,12 +722,12 @@ proc processPragma(c: PContext, n: PNode, i: int) =
 proc pragmaRaisesOrTags(c: PContext, n: PNode) =
   proc processExc(c: PContext, x: PNode) =
     if c.hasUnresolvedArgs(c, x):
-      x.typ = makeTypeFromExpr(c, x)
+      x.typ() = makeTypeFromExpr(c, x)
     else:
       var t = skipTypes(c.semTypeNode(c, x, nil), skipPtrs)
       if t.kind notin {tyObject, tyOr}:
         localError(c.config, x.info, errGenerated, "invalid type for raises/tags list")
-      x.typ = t
+      x.typ() = t
 
   if n.kind in nkPragmaCallKinds and n.len == 2:
     let it = n[1]
@@ -787,13 +800,14 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =
 proc semCustomPragma(c: PContext, n: PNode, sym: PSym): PNode =
   var callNode: PNode
 
-  if n.kind in {nkIdent, nkSym}:
+  case n.kind
+  of nkIdentKinds:
     # pragma -> pragma()
     callNode = newTree(nkCall, n)
-  elif n.kind == nkExprColonExpr:
+  of nkExprColonExpr:
     # pragma: arg -> pragma(arg)
     callNode = newTree(nkCall, n[0], n[1])
-  elif n.kind in nkPragmaCallKinds:
+  of nkPragmaCallKinds - {nkExprColonExpr}:
     callNode = n
   else:
     invalidPragma(c, n)
@@ -869,6 +883,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
 
     # number of pragmas increase/decrease with user pragma expansion
     inc c.instCounter
+    defer: dec c.instCounter
     if c.instCounter > 100:
       globalError(c.config, it.info, "recursive dependency: " & userPragma.name.s)
 
@@ -878,7 +893,6 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
     pragma(c, sym, userPragma.ast, validPragmas, isStatement)
     n.sons[i..i] = userPragma.ast.sons # expand user pragma with its content
     i.inc(userPragma.ast.len - 1) # inc by -1 is ok, user pragmas was empty
-    dec c.instCounter
   else:
     let k = whichKeyword(ident)
     if k in validPragmas:
@@ -1001,7 +1015,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         incl(sym.loc.flags, lfHeader)
         incl(sym.loc.flags, lfNoDecl)
         # implies nodecl, because otherwise header would not make sense
-        if sym.loc.r == "": sym.loc.r = rope(sym.name.s)
+        if sym.loc.snippet == "": sym.loc.snippet = rope(sym.name.s)
       of wNoSideEffect:
         noVal(c, it)
         if sym != nil:
@@ -1295,7 +1309,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         noVal(c, it)
         if sym == nil: invalidPragma(c, it)
         else: sym.flags.incl sfUsed
-      of wLiftLocals: discard
+      of wLiftLocals:
+        sym.flags.incl(sfForceLift)
       of wRequires, wInvariant, wAssume, wAssert:
         pragmaProposition(c, it)
       of wEnsures:
@@ -1329,6 +1344,16 @@ proc mergePragmas(n, pragmas: PNode) =
   else:
     for p in pragmas: n[pragmasPos].add p
 
+proc mergeValidPragmas(n, pragmas: PNode, validPragmas: TSpecialWords) =
+  if n[pragmasPos].kind == nkEmpty:
+    n[pragmasPos] = newNodeI(nkPragma, n.info)
+  for p in pragmas:
+    let prag = whichPragma(p)
+    if prag in validPragmas:
+      let copy = copyTree(p)
+      overwriteLineInfo copy, n.info
+      n[pragmasPos].add copy
+
 proc implicitPragmas*(c: PContext, sym: PSym, info: TLineInfo,
                       validPragmas: TSpecialWords) =
   if sym != nil and sym.kind != skModule:
@@ -1342,7 +1367,8 @@ proc implicitPragmas*(c: PContext, sym: PSym, info: TLineInfo,
             internalError(c.config, info, "implicitPragmas")
           inc i
         popInfoContext(c.config)
-        if sym.kind in routineKinds and sym.ast != nil: mergePragmas(sym.ast, o)
+        if sym.kind in routineKinds and sym.ast != nil:
+          mergeValidPragmas(sym.ast, o, validPragmas)
 
     if lfExportLib in sym.loc.flags and sfExportc notin sym.flags:
       localError(c.config, info, ".dynlib requires .exportc")
@@ -1351,7 +1377,7 @@ proc implicitPragmas*(c: PContext, sym: PSym, info: TLineInfo,
         sfImportc in sym.flags and lib != nil:
       incl(sym.loc.flags, lfDynamicLib)
       addToLib(lib, sym)
-      if sym.loc.r == "": sym.loc.r = rope(sym.name.s)
+      if sym.loc.snippet == "": sym.loc.snippet = rope(sym.name.s)
 
 proc hasPragma*(n: PNode, pragma: TSpecialWord): bool =
   if n == nil: return false

compiler/pushpoppragmas.nim

@@ -0,0 +1,54 @@
+import pragmas, options, ast, trees, lineinfos, idents, wordrecg
+import std/assertions
+
+import renderer
+
+
+proc processNote(config: ConfigRef, n: PNode) =
+  template handleNote(enumVals, notes) =
+    let x = findStr(enumVals.a, enumVals.b, n[0][1].ident.s, errUnknown)
+    assert x != errUnknown
+    assert n[1].kind == nkIntLit
+
+    nk = TNoteKind(x)
+    if n[1].intVal != 0: incl(notes, nk)
+    else: excl(notes, nk)
+
+  var nk: TNoteKind
+  case whichKeyword(n[0][0].ident)
+  of wHint: handleNote(hintMin .. hintMax, config.notes)
+  of wWarning: handleNote(warnMin .. warnMax, config.notes)
+  of wWarningAsError: handleNote(warnMin .. warnMax, config.warningAsErrors)
+  of wHintAsError: handleNote(hintMin .. hintMax, config.warningAsErrors)
+  else: discard
+
+proc pushBackendOption(optionsStack: var seq[(TOptions, TNoteKinds)], options: TOptions, notes: TNoteKinds) =
+  optionsStack.add (options, notes)
+
+proc popBackendOption(config: ConfigRef, optionsStack: var seq[(TOptions, TNoteKinds)], options: var TOptions) =
+  let entry = optionsStack[^1]
+  options = entry[0]
+  config.notes = entry[1]
+  optionsStack.setLen(optionsStack.len-1)
+
+proc processPushBackendOption*(config: ConfigRef, optionsStack: var seq[(TOptions, TNoteKinds)], options: var TOptions,
+                           n: PNode, start: int) =
+  pushBackendOption(optionsStack, options, config.notes)
+  for i in start..<n.len:
+    let it = n[i]
+    if it.kind in nkPragmaCallKinds and it.len == 2:
+      if it[0].kind == nkBracketExpr and
+          it[0].len == 2 and
+          it[0][1].kind == nkIdent and it[0][0].kind == nkIdent:
+        processNote(config, it)
+      elif it[1].kind == nkIntLit:
+        let sw = whichPragma(it[0])
+        let opts = pragmaToOptions(sw)
+        if opts != {}:
+          if it[1].intVal != 0:
+            options.incl opts
+          else:
+            options.excl opts
+
+template processPopBackendOption*(config: ConfigRef, optionsStack: var seq[(TOptions, TNoteKinds)], options: var TOptions) =
+  popBackendOption(config, optionsStack, options)

compiler/renderer.nim

@@ -442,6 +442,11 @@ proc atom(g: TSrcGen; n: PNode): string =
       result = $n.floatVal & "\'f64"
     else:
       result = litAux(g, n, (cast[ptr int64](addr(n.floatVal)))[], 8) & "\'f64"
+  of nkFloat128Lit:
+    if n.flags * {nfBase2, nfBase8, nfBase16} == {}:
+      result = $n.floatVal & "\'f128"
+    else:
+      result = litAux(g, n, (cast[ptr int64](addr(n.floatVal)))[], 8) & "\'f128"
   of nkNilLit: result = "nil"
   of nkType:
     if (n.typ != nil) and (n.typ.sym != nil): result = n.typ.sym.name.s
@@ -514,6 +519,7 @@ proc lsub(g: TSrcGen; n: PNode): int =
     result = if n.len > 0: lcomma(g, n) + 2 else: len("{:}")
   of nkClosedSymChoice, nkOpenSymChoice:
     if n.len > 0: result += lsub(g, n[0])
+  of nkOpenSym: result = lsub(g, n[0])
   of nkTupleTy: result = lcomma(g, n) + len("tuple[]")
   of nkTupleClassTy: result = len("tuple")
   of nkDotExpr: result = lsons(g, n) + 1
@@ -1013,7 +1019,7 @@ proc bracketKind*(g: TSrcGen, n: PNode): BracketKind =
 proc skipHiddenNodes(n: PNode): PNode =
   result = n
   while result != nil:
-    if result.kind in {nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv} and result.len > 1:
+    if result.kind in {nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv, nkOpenSym} and result.len > 1:
       result = result[1]
     elif result.kind in {nkCheckedFieldExpr, nkHiddenAddr, nkHiddenDeref, nkStringToCString, nkCStringToString} and
         result.len > 0:
@@ -1275,6 +1281,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
       put(g, tkParRi, if n.kind == nkOpenSymChoice: "|...)" else: ")")
     else:
       gsub(g, n, 0)
+  of nkOpenSym: gsub(g, n, 0)
   of nkPar, nkClosure:
     put(g, tkParLe, "(")
     gcomma(g, n, c)
@@ -1304,10 +1311,11 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
       put(g, tkCustomLit, n[0].strVal)
       gsub(g, n, 1)
     else:
-      gsub(g, n, 0)
+      for i in 0..<n.len-1:
+        gsub(g, n, i)
       put(g, tkDot, ".")
-      assert n.len == 2, $n.len
-      accentedName(g, n[1])
+      if n.len > 1:
+        accentedName(g, n[^1])
   of nkBind:
     putWithSpace(g, tkBind, "bind")
     gsub(g, n, 0)

compiler/reorder.nim

@@ -322,6 +322,14 @@ proc intersects(s1, s2: IntSet): bool =
     if s2.contains(a):
       return true
 
+proc hasPushOrPopPragma(n: DepN): bool =
+  # Checks if the tree node has some pragmas that do not
+  # play well with reordering, like the push/pop pragma
+  # no crossing for push/pop barrier
+  let a = n.pnode
+  result = a.kind == nkPragma and a[0].kind == nkIdent and
+      (a[0].ident.s == "push" or a[0].ident.s == "pop")
+
 proc buildGraph(n: PNode, deps: seq[(IntSet, IntSet)]): DepG =
   # Build a dependency graph
   result = newSeqOfCap[DepN](deps.len)
@@ -363,6 +371,13 @@ proc buildGraph(n: PNode, deps: seq[(IntSet, IntSet)]): DepG =
             for dep in deps[i][0]:
               if dep in declares:
                 ni.expls.add "one declares \"" & idNames[dep] & "\" and the other defines it"
+      elif hasPushOrPopPragma(nj):
+        # Every node that comes after a push/pop pragma must
+        # depend on it; vice versa
+        if j < i:
+          ni.kids.add nj
+        else:
+          nj.kids.add ni
       else:
         for d in declares:
           if uses.contains(d):
@@ -403,18 +418,7 @@ proc getStrongComponents(g: var DepG): seq[seq[DepN]] =
     if v.idx < 0:
       strongConnect(v, idx, s, result)
 
-proc hasForbiddenPragma(n: PNode): bool =
-  # Checks if the tree node has some pragmas that do not
-  # play well with reordering, like the push/pop pragma
-  result = false
-  for a in n:
-    if a.kind == nkPragma and a[0].kind == nkIdent and
-        a[0].ident.s == "push":
-      return true
-
 proc reorder*(graph: ModuleGraph, n: PNode, module: PSym): PNode =
-  if n.hasForbiddenPragma:
-    return n
   var includedFiles = initIntSet()
   let mpath = toFullPath(graph.config, module.fileIdx)
   let n = expandIncludes(graph, module, n, mpath,

compiler/sem.nim

@@ -18,10 +18,10 @@ import
   evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity,
   lowerings, plugins/active, lineinfos, int128,
   isolation_check, typeallowed, modulegraphs, enumtostr, concepts, astmsgs,
-  extccomp
+  extccomp, layeredtable
 
 import vtables
-import std/[strtabs, math, tables, intsets, strutils]
+import std/[strtabs, math, tables, intsets, strutils, packedsets]
 
 when not defined(leanCompiler):
   import spawn
@@ -89,6 +89,11 @@ proc fitNodePostMatch(c: PContext, formal: PType, arg: PNode): PNode =
     changeType(c, x, formal, check=true)
   result = arg
   result = skipHiddenSubConv(result, c.graph, c.idgen)
+  # mark inserted converter as used:
+  var a = result
+  if a.kind == nkHiddenDeref: a = a[0]
+  if a.kind == nkHiddenCallConv and a[0].kind == nkSym:
+    markUsed(c, a.info, a[0].sym)
 
 
 proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
@@ -97,13 +102,13 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
                renderTree(arg, {renderNoComments}))
     # error correction:
     result = copyTree(arg)
-    result.typ = formal
+    result.typ() = formal
   elif arg.kind in nkSymChoices and formal.skipTypes(abstractInst).kind == tyEnum:
     # Pick the right 'sym' from the sym choice by looking at 'formal' type:
     result = nil
     for ch in arg:
       if sameType(ch.typ, formal):
-        return getConstExpr(c.module, ch, c.idgen, c.graph)
+        return ch
     typeMismatch(c.config, info, formal, arg.typ, arg)
   else:
     result = indexTypesMatch(c, formal, arg.typ, arg)
@@ -111,7 +116,7 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
       typeMismatch(c.config, info, formal, arg.typ, arg)
       # error correction:
       result = copyTree(arg)
-      result.typ = formal
+      result.typ() = formal
     else:
       result = fitNodePostMatch(c, formal, result)
 
@@ -222,66 +227,7 @@ proc shouldCheckCaseCovered(caseTyp: PType): bool =
   else:
     discard
 
-proc endsInNoReturn(n: PNode): bool =
-  ## check if expr ends the block like raising or call of noreturn procs do
-  result = false # assume it does return
-
-  template checkBranch(branch) =
-    if not endsInNoReturn(branch):
-      # proved a branch returns
-      return false
-
-  var it = n
-  # skip these beforehand, no special handling needed
-  while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0:
-    it = it.lastSon
-
-  case it.kind
-  of nkIfStmt:
-    var hasElse = false
-    for branch in it:
-      checkBranch:
-        if branch.len == 2:
-          branch[1]
-        elif branch.len == 1:
-          hasElse = true
-          branch[0]
-        else:
-          raiseAssert "Malformed `if` statement during endsInNoReturn"
-    # none of the branches returned
-    result = hasElse # Only truly a no-return when it's exhaustive
-  of nkCaseStmt:
-    let caseTyp = skipTypes(it[0].typ, abstractVar-{tyTypeDesc})
-    # semCase should already have checked for exhaustiveness in this case
-    # effectively the same as having an else
-    var hasElse = caseTyp.shouldCheckCaseCovered()
-
-    # actual noreturn checks
-    for i in 1 ..< it.len:
-      let branch = it[i]
-      checkBranch:
-        case branch.kind
-        of nkOfBranch:
-          branch[^1]
-        of nkElifBranch:
-          branch[1]
-        of nkElse:
-          hasElse = true
-          branch[0]
-        else:
-          raiseAssert "Malformed `case` statement in endsInNoReturn"
-    # Can only guarantee a noreturn if there is an else or it's exhaustive
-    result = hasElse
-  of nkTryStmt:
-    checkBranch(it[0])
-    for i in 1 ..< it.len:
-      let branch = it[i]
-      checkBranch(branch[^1])
-    # none of the branches returned
-    result = true
-  else:
-    result = it.kind in nkLastBlockStmts or
-      it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags
+proc endsInNoReturn(n: PNode): bool
 
 proc commonType*(c: PContext; x: PType, y: PNode): PType =
   # ignore exception raising branches in case/if expressions
@@ -310,9 +256,11 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     # when there is a nested proc inside a template, semtmpl
     # will assign a wrong owner during the first pass over the
     # template; we must fix it here: see #909
-    result.owner = getCurrOwner(c)
+    setOwner(result, getCurrOwner(c))
   else:
     result = newSym(kind, considerQuotedIdent(c, n), c.idgen, getCurrOwner(c), n.info)
+    if find(result.name.s, '`') >= 0:
+      result.flags.incl sfWasGenSym
   #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule:
   #  incl(result.flags, sfGlobal)
   when defined(nimsuggest):
@@ -322,7 +270,7 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
                  allowed: TSymFlags): PSym
   # identifier with visibility
 proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
-                        allowed: TSymFlags): PSym
+                        allowed: TSymFlags, fromTopLevel = false): PSym
 
 proc typeAllowedCheck(c: PContext; info: TLineInfo; typ: PType; kind: TSymKind;
                       flags: TTypeAllowedFlags = {}) =
@@ -522,7 +470,7 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
                    renderTree(result, {renderNoComments}))
         result = newSymNode(errorSym(c, result))
       else:
-        result.typ = makeTypeDesc(c, typ)
+        result.typ() = makeTypeDesc(c, typ)
       #result = symNodeFromType(c, typ, n.info)
     else:
       if s.ast[genericParamsPos] != nil and retType.isMetaType:
@@ -530,7 +478,7 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
         # e.g. template foo(T: typedesc): seq[T]
         # We will instantiate the return type here, because
         # we now know the supplied arguments
-        var paramTypes = initTypeMapping()
+        var paramTypes = initLayeredTypeMap()
         for param, value in genericParamsInMacroCall(s, call):
           var givenType = value.typ
           # the sym nodes used for the supplied generic arguments for
@@ -538,7 +486,7 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
           # in this case, get the type directly from the sym
           if givenType == nil and value.kind == nkSym and value.sym.typ != nil:
             givenType = value.sym.typ
-          idTablePut(paramTypes, param.typ, givenType)
+          put(paramTypes, param.typ, givenType)
 
         retType = generateTypeInstance(c, paramTypes,
                                        macroResult.info, retType)
@@ -546,7 +494,7 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
       if retType.kind == tyVoid:
         result = semStmt(c, result, flags)
       else:
-        result = semExpr(c, result, flags, expectedType)
+        result = semExpr(c, result, flags)
         result = fitNode(c, retType, result, result.info)
       #globalError(s.info, errInvalidParamKindX, typeToString(s.typ.returnType))
   dec(c.config.evalTemplateCounter)
@@ -554,7 +502,6 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
 
 const
   errMissingGenericParamsForTemplate = "'$1' has unspecified generic parameters"
-  errFloatToString = "cannot convert '$1' to '$2'"
 
 proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
                   flags: TExprFlags = {}; expectedType: PType = nil): PNode =
@@ -665,7 +612,7 @@ proc defaultFieldsForTuple(c: PContext, recNode: PNode, hasDefault: var bool, ch
                       newNodeIT(nkType, recNode.info, asgnType)
                     )
       asgnExpr.flags.incl nfSkipFieldChecking
-      asgnExpr.typ = recNode.typ
+      asgnExpr.typ() = recNode.typ
       result.add newTree(nkExprColonExpr, recNode, asgnExpr)
   else:
     raiseAssert "unreachable"
@@ -687,7 +634,7 @@ proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, checkDefault:
       if checkDefault: # don't add defaults when checking whether a case branch has default fields
         return
       defaultValue = newIntNode(nkIntLit#[c.graph]#, 0)
-      defaultValue.typ = discriminator.typ
+      defaultValue.typ() = discriminator.typ
     selectedBranch = recNode.pickCaseBranchIndex defaultValue
     defaultValue.flags.incl nfSkipFieldChecking
     result.add newTree(nkExprColonExpr, discriminator, defaultValue)
@@ -700,7 +647,7 @@ proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, checkDefault:
     elif recType.kind in {tyObject, tyArray, tyTuple}:
       let asgnExpr = defaultNodeField(c, recNode, recNode.typ, checkDefault)
       if asgnExpr != nil:
-        asgnExpr.typ = recNode.typ
+        asgnExpr.typ() = recNode.typ
         asgnExpr.flags.incl nfSkipFieldChecking
         result.add newTree(nkExprColonExpr, recNode, asgnExpr)
   else:
@@ -708,16 +655,17 @@ proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, checkDefault:
 
 proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, checkDefault: bool): PNode =
   let aTypSkip = aTyp.skipTypes(defaultFieldsSkipTypes)
-  if aTypSkip.kind == tyObject:
+  case aTypSkip.kind
+  of tyObject:
     let child = defaultFieldsForTheUninitialized(c, aTypSkip.n, checkDefault)
     if child.len > 0:
       var asgnExpr = newTree(nkObjConstr, newNodeIT(nkType, a.info, aTyp))
-      asgnExpr.typ = aTyp
+      asgnExpr.typ() = aTyp
       asgnExpr.sons.add child
       result = semExpr(c, asgnExpr)
     else:
       result = nil
-  elif aTypSkip.kind == tyArray:
+  of tyArray:
     let child = defaultNodeField(c, a, aTypSkip[1], checkDefault)
 
     if child != nil:
@@ -727,22 +675,27 @@ proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, checkDefault: bool): P
               semExprWithType(c, child),
               node
                 ))
-      result.typ = aTyp
+      result.typ() = aTyp
     else:
       result = nil
-  elif aTypSkip.kind == tyTuple:
+  of tyTuple:
     var hasDefault = false
     if aTypSkip.n != nil:
       let children = defaultFieldsForTuple(c, aTypSkip.n, hasDefault, checkDefault)
       if hasDefault and children.len > 0:
         result = newNodeI(nkTupleConstr, a.info)
-        result.typ = aTyp
+        result.typ() = aTyp
         result.sons.add children
         result = semExpr(c, result)
       else:
         result = nil
     else:
       result = nil
+  of tyRange:
+    if c.graph.config.isDefined("nimPreviewRangeDefault"):
+      result = firstRange(c.config, aTypSkip)
+    else:
+      result = nil
   else:
     result = nil
 
@@ -779,6 +732,8 @@ proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PCo
   result.semOverloadedCall = semOverloadedCall
   result.semInferredLambda = semInferredLambda
   result.semGenerateInstance = generateInstance
+  result.instantiateOnlyProcType = instantiateOnlyProcType
+  result.fitDefaultNode = fitDefaultNode
   result.semTypeNode = semTypeNode
   result.instTypeBoundOp = sigmatch.instTypeBoundOp
   result.hasUnresolvedArgs = hasUnresolvedArgs

compiler/semcall.nim

@@ -69,6 +69,39 @@ proc initCandidateSymbols(c: PContext, headSymbol: PNode,
                   result[0].scope, diagnostics)
     best.state = csNoMatch
 
+proc isAttachableRoutineTo(prc: PSym, arg: PType): bool =
+  result = false
+  if arg.owner != prc.owner: return false
+  for i in 1 ..< prc.typ.len:
+    if prc.typ.n[i].kind == nkSym and prc.typ.n[i].sym.ast != nil:
+      # has default value, parameter is not considered in type attachment
+      continue
+    let t = nominalRoot(prc.typ[i])
+    if t != nil and t.itemId == arg.itemId:
+      # parameter `i` is a nominal type in this module
+      # attachable if the nominal root `t` has the same id as `arg`
+      return true
+
+proc addTypeBoundSymbols(graph: ModuleGraph, arg: PType, name: PIdent,
+                         filter: TSymKinds, marker: var IntSet,
+                         syms: var seq[tuple[s: PSym, scope: int]]) =
+  # add type bound ops for `name` based on the argument type `arg`
+  if arg != nil:
+    # argument must be typed first, meaning arguments always
+    # matching `untyped` are ignored
+    let t = nominalRoot(arg)
+    if t != nil and t.owner.kind == skModule:
+      # search module for routines attachable to `t`
+      let module = t.owner
+      var iter = default(ModuleIter)
+      var s = initModuleIter(iter, graph, module, name)
+      while s != nil:
+        if s.kind in filter and s.isAttachableRoutineTo(t) and
+            not containsOrIncl(marker, s.id):
+          # least priority scope, less than explicit imports:
+          syms.add((s, -2))
+        s = nextModuleIter(iter, graph)
+
 proc pickBestCandidate(c: PContext, headSymbol: PNode,
                        n, orig: PNode,
                        initialBinding: PNode,
@@ -88,10 +121,23 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
                                   best, alt, o, diagnosticsFlag)
   if len(syms) == 0:
     return
+  let allowTypeBoundOps = typeBoundOps in c.features and
+    # qualified or bound symbols cannot refer to type bound ops
+    headSymbol.kind in {nkIdent, nkAccQuoted, nkOpenSymChoice, nkOpenSym}
+  var symMarker = initIntSet()
+  for s in syms:
+    symMarker.incl(s.s.id)
   # current overload being considered
   var sym = syms[0].s
+  let name = sym.name
   var scope = syms[0].scope
 
+  if allowTypeBoundOps:
+    for a in 1 ..< n.len:
+      # for every already typed argument, add type bound ops
+      let arg = n[a]
+      addTypeBoundSymbols(c.graph, arg.typ, name, filter, symMarker, syms)
+
   # starts at 1 because 0 is already done with setup, only needs checking
   var nextSymIndex = 1
   var z: TCandidate # current candidate
@@ -106,6 +152,14 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
       # may introduce new symbols with caveats described in recalc branch
       matches(c, n, orig, z)
 
+      if allowTypeBoundOps:
+        # this match may have given some arguments new types,
+        # in which case add their type bound ops as well
+        # type bound ops of arguments always matching `untyped` are not considered
+        for x in z.newlyTypedOperands:
+          let arg = n[x]
+          addTypeBoundSymbols(c.graph, arg.typ, name, filter, symMarker, syms)
+
       if z.state == csMatch:
         # little hack so that iterators are preferred over everything else:
         if sym.kind == skIterator:
@@ -136,7 +190,14 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
       # before any further candidate init and compare. SLOW, but rare case.
       syms = initCandidateSymbols(c, headSymbol, initialBinding, filter,
                                   best, alt, o, diagnosticsFlag)
-
+      symMarker = initIntSet()
+      for s in syms:
+        symMarker.incl(s.s.id)
+      if allowTypeBoundOps:
+        for a in 1 ..< n.len:
+          # for every already typed argument, add type bound ops
+          let arg = n[a]
+          addTypeBoundSymbols(c.graph, arg.typ, name, filter, symMarker, syms)
       # reset counter because syms may be in a new order
       symCount = c.currentScope.symbols.counter
       nextSymIndex = 0
@@ -246,7 +307,16 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
       candidates.add(getProcHeader(c.config, err.sym, prefer))
     candidates.addDeclaredLocMaybe(c.config, err.sym)
     candidates.add("\n")
-    let nArg = if err.firstMismatch.arg < n.len: n[err.firstMismatch.arg] else: nil
+    const genericParamMismatches = {kGenericParamTypeMismatch, kExtraGenericParam, kMissingGenericParam}
+    let isGenericMismatch = err.firstMismatch.kind in genericParamMismatches
+    var argList = n
+    if isGenericMismatch and n[0].kind == nkBracketExpr:
+      argList = n[0]
+    let nArg =
+      if err.firstMismatch.arg < argList.len:
+        argList[err.firstMismatch.arg]
+      else:
+        nil
     let nameParam = if err.firstMismatch.formal != nil: err.firstMismatch.formal.name.s else: ""
     if n.len > 1:
       if verboseTypeMismatch notin c.config.legacyFeatures:
@@ -269,6 +339,12 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
         of kMissingParam:
           candidates.add("  missing parameter: " & nameParam)
           candidates.add "\n"
+        of kExtraGenericParam:
+          candidates.add("  extra generic param given")
+          candidates.add "\n"
+        of kMissingGenericParam:
+          candidates.add("  missing generic parameter: " & nameParam)
+          candidates.add "\n"
         of kVarNeeded:
           doAssert nArg != nil
           doAssert err.firstMismatch.formal != nil
@@ -278,6 +354,8 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
           candidates.add "\n"
         of kTypeMismatch:
           doAssert nArg != nil
+          if nArg.kind in nkSymChoices:
+            candidates.add ambiguousIdentifierMsg(nArg, indent = 2)
           let wanted = err.firstMismatch.formal.typ
           doAssert err.firstMismatch.formal != nil
           doAssert wanted != nil
@@ -292,9 +370,39 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
             candidates.addPragmaAndCallConvMismatch(wanted, got, c.config)
             effectProblem(wanted, got, candidates, c)
             candidates.add "\n"
+        of kGenericParamTypeMismatch:
+          let pos = err.firstMismatch.arg
+          doAssert n[0].kind == nkBracketExpr and pos < n[0].len
+          let arg = n[0][pos]
+          doAssert arg != nil
+          var wanted = err.firstMismatch.formal.typ
+          if wanted.kind == tyGenericParam and wanted.genericParamHasConstraints:
+            wanted = wanted.genericConstraint
+          let got = arg.typ.skipTypes({tyTypeDesc})
+          doAssert err.firstMismatch.formal != nil
+          doAssert wanted != nil
+          doAssert got != nil
+          candidates.add "  generic parameter mismatch, expected "
+          candidates.addTypeDeclVerboseMaybe(c.config, wanted)
+          candidates.add " but got '"
+          candidates.add renderTree(arg)
+          candidates.add "' of type: "
+          candidates.addTypeDeclVerboseMaybe(c.config, got)
+          if nArg.kind in nkSymChoices:
+            candidates.add "\n"
+            candidates.add ambiguousIdentifierMsg(nArg, indent = 2)
+          if got != nil and got.kind == tyProc and wanted.kind == tyProc:
+            # These are proc mismatches so,
+            # add the extra explict detail of the mismatch
+            candidates.addPragmaAndCallConvMismatch(wanted, got, c.config)
+          if got != nil:
+            effectProblem(wanted, got, candidates, c)
+          candidates.add "\n"
         of kUnknown: discard "do not break 'nim check'"
       else:
         candidates.add("  first type mismatch at position: " & $err.firstMismatch.arg)
+        if err.firstMismatch.kind in genericParamMismatches:
+          candidates.add(" in generic parameters")
         # candidates.add "\n  reason: " & $err.firstMismatch.kind # for debugging
         case err.firstMismatch.kind
         of kUnknownNamedParam:
@@ -306,9 +414,16 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
         of kPositionalAlreadyGiven: candidates.add("\n  positional param was already given as named param")
         of kExtraArg: candidates.add("\n  extra argument given")
         of kMissingParam: candidates.add("\n  missing parameter: " & nameParam)
-        of kTypeMismatch, kVarNeeded:
+        of kExtraGenericParam:
+          candidates.add("\n  extra generic param given")
+        of kMissingGenericParam:
+          candidates.add("\n  missing generic parameter: " & nameParam)
+        of kTypeMismatch, kGenericParamTypeMismatch, kVarNeeded:
           doAssert nArg != nil
-          let wanted = err.firstMismatch.formal.typ
+          var wanted = err.firstMismatch.formal.typ
+          if isGenericMismatch and wanted.kind == tyGenericParam and
+              wanted.genericParamHasConstraints:
+            wanted = wanted.genericConstraint
           doAssert err.firstMismatch.formal != nil
           candidates.add("\n  required type for " & nameParam &  ": ")
           candidates.addTypeDeclVerboseMaybe(c.config, wanted)
@@ -319,8 +434,12 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
           else:
             candidates.add renderTree(nArg)
             candidates.add "' is of type: "
-            let got = nArg.typ
+            var got = nArg.typ
+            if isGenericMismatch: got = got.skipTypes({tyTypeDesc})
             candidates.addTypeDeclVerboseMaybe(c.config, got)
+            if nArg.kind in nkSymChoices:
+              candidates.add "\n"
+              candidates.add ambiguousIdentifierMsg(nArg, indent = 2)
             doAssert wanted != nil
             if got != nil:
               if got.kind == tyProc and wanted.kind == tyProc:
@@ -331,8 +450,8 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
 
         of kUnknown: discard "do not break 'nim check'"
         candidates.add "\n"
-      if err.firstMismatch.arg == 1 and nArg.kind == nkTupleConstr and
-          n.kind == nkCommand:
+      if err.firstMismatch.arg == 1 and nArg != nil and
+          nArg.kind == nkTupleConstr and n.kind == nkCommand:
         maybeWrongSpace = true
     for diag in err.diagnostics:
       candidates.add(diag & "\n")
@@ -409,23 +528,6 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) =
       result.add("\n" & errExpectedPosition & "\n" & candidates)
     localError(c.config, n.info, result)
 
-proc bracketNotFoundError(c: PContext; n: PNode) =
-  var errors: CandidateErrors = @[]
-  var o: TOverloadIter = default(TOverloadIter)
-  let headSymbol = n[0]
-  var symx = initOverloadIter(o, c, headSymbol)
-  while symx != nil:
-    if symx.kind in routineKinds:
-      errors.add(CandidateError(sym: symx,
-                                firstMismatch: MismatchInfo(),
-                                diagnostics: @[],
-                                enabled: false))
-    symx = nextOverloadIter(o, c, headSymbol)
-  if errors.len == 0:
-    localError(c.config, n.info, "could not resolve: " & $n)
-  else:
-    notFoundError(c, n, errors)
-
 proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string =
   result = ""
   if c.compilesContextId > 0:
@@ -518,7 +620,8 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
     if overloadsState == csEmpty and result.state == csEmpty:
       if efNoUndeclared notin flags: # for tests/pragmas/tcustom_pragma.nim
         result.state = csNoMatch
-        if efNoDiagnostics in flags:
+        if c.inGenericContext > 0 and nfExprCall in n.flags:
+          # untyped expression calls end up here, see #24099
           return
         # xxx adapt/use errorUndeclaredIdentifierHint(c, n, f.ident)
         localError(c.config, n.info, getMsgDiagnostic(c, flags, n, f))
@@ -554,6 +657,39 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
         getProcHeader(c.config, alt.calleeSym),
         args])
 
+proc bracketNotFoundError(c: PContext; n: PNode; flags: TExprFlags) =
+  var errors: CandidateErrors = @[]
+  let headSymbol = n[0]
+  block:
+    # we build a closed symchoice of all `[]` overloads for their errors,
+    # except add a custom error for the magics which always match
+    var choice = newNodeIT(nkClosedSymChoice, headSymbol.info, newTypeS(tyNone, c))
+    var o: TOverloadIter = default(TOverloadIter)
+    var symx = initOverloadIter(o, c, headSymbol)
+    while symx != nil:
+      if symx.kind in routineKinds:
+        if symx.magic in {mArrGet, mArrPut}:
+          errors.add(CandidateError(sym: symx,
+                                    firstMismatch: MismatchInfo(),
+                                    diagnostics: @[],
+                                    enabled: false))
+        else:
+          choice.add newSymNode(symx, headSymbol.info)
+      symx = nextOverloadIter(o, c, headSymbol)
+    n[0] = choice
+  # copied from semOverloadedCallAnalyzeEffects, might be overkill:
+  const baseFilter = {skProc, skFunc, skMethod, skConverter, skMacro, skTemplate}
+  let filter =
+    if flags*{efInTypeof, efWantIterator, efWantIterable} != {}:
+      baseFilter + {skIterator}
+    else: baseFilter
+  # this will add the errors:
+  var r = resolveOverloads(c, n, n, filter, flags, errors, true)
+  if errors.len == 0:
+    localError(c.config, n.info, "could not resolve: " & $n)
+  else:
+    notFoundError(c, n, errors)
+
 proc instGenericConvertersArg*(c: PContext, a: PNode, x: TCandidate) =
   let a = if a.kind == nkHiddenDeref: a[0] else: a
   if a.kind == nkHiddenCallConv and a[0].kind == nkSym:
@@ -561,7 +697,7 @@ proc instGenericConvertersArg*(c: PContext, a: PNode, x: TCandidate) =
     if s.isGenericRoutineStrict:
       let finalCallee = generateInstance(c, s, x.bindings, a.info)
       a[0].sym = finalCallee
-      a[0].typ = finalCallee.typ
+      a[0].typ() = finalCallee.typ
       #a.typ = finalCallee.typ.returnType
 
 proc instGenericConvertersSons*(c: PContext, n: PNode, x: TCandidate) =
@@ -570,6 +706,15 @@ proc instGenericConvertersSons*(c: PContext, n: PNode, x: TCandidate) =
     for i in 1..<n.len:
       instGenericConvertersArg(c, n[i], x)
 
+proc markConvertersUsed*(c: PContext, n: PNode) =
+  assert n.kind in nkCallKinds
+  for i in 1..<n.len:
+    var a = n[i]
+    if a == nil: continue
+    if a.kind == nkHiddenDeref: a = a[0]
+    if a.kind == nkHiddenCallConv and a[0].kind == nkSym:
+      markUsed(c, a.info, a[0].sym)
+
 proc indexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode =
   var m = newCandidate(c, f)
   result = paramTypesMatch(m, f, a, arg, nil)
@@ -587,15 +732,15 @@ proc inferWithMetatype(c: PContext, formal: PType,
     # This almost exactly replicates the steps taken by the compiler during
     # param matching. It performs an embarrassing amount of back-and-forth
     # type jugling, but it's the price to pay for consistency and correctness
-    result.typ = generateTypeInstance(c, m.bindings, arg.info,
+    result.typ() = generateTypeInstance(c, m.bindings, arg.info,
                                       formal.skipTypes({tyCompositeTypeClass}))
   else:
     typeMismatch(c.config, arg.info, formal, arg.typ, arg)
     # error correction:
     result = copyTree(arg)
-    result.typ = formal
+    result.typ() = formal
 
-proc updateDefaultParams(call: PNode) =
+proc updateDefaultParams(c: PContext, call: PNode) =
   # In generic procs, the default parameter may be unique for each
   # instantiation (see tlateboundgenericparams).
   # After a call is resolved, we need to re-assign any default value
@@ -605,8 +750,18 @@ proc updateDefaultParams(call: PNode) =
   let calleeParams = call[0].sym.typ.n
   for i in 1..<call.len:
     if nfDefaultParam in call[i].flags:
-      let def = calleeParams[i].sym.ast
+      let formal = calleeParams[i].sym
+      let def = formal.ast
       if nfDefaultRefsParam in def.flags: call.flags.incl nfDefaultRefsParam
+      # mirrored with sigmatch:
+      if def.kind == nkEmpty:
+        # The default param value is set to empty in `instantiateProcType`
+        # when the type of the default expression doesn't match the type
+        # of the instantiated proc param:
+        pushInfoContext(c.config, call.info, call[0].sym.detailedInfo)
+        typeMismatch(c.config, def.info, formal.typ, def.typ, formal.ast)
+        popInfoContext(c.config)
+        def.typ() = errorType(c)
       call[i] = def
 
 proc getCallLineInfo(n: PNode): TLineInfo =
@@ -663,7 +818,7 @@ proc inheritBindings(c: PContext, x: var TCandidate, expectedType: PType) =
         if t[i] == nil or u[i] == nil: return
         stackPut(t[i], u[i])
     of tyGenericParam:
-      let prebound = x.bindings.idTableGet(t)
+      let prebound = x.bindings.lookup(t)
       if prebound != nil:
         continue # Skip param, already bound
 
@@ -675,7 +830,7 @@ proc inheritBindings(c: PContext, x: var TCandidate, expectedType: PType) =
       discard
   # update bindings
   for i in 0 ..< flatUnbound.len():
-    x.bindings.idTablePut(flatUnbound[i], flatBound[i])
+    x.bindings.put(flatUnbound[i], flatBound[i])
 
 proc semResolvedCall(c: PContext, x: var TCandidate,
                      n: PNode, flags: TExprFlags;
@@ -686,12 +841,12 @@ proc semResolvedCall(c: PContext, x: var TCandidate,
   markUsed(c, info, finalCallee)
   onUse(info, finalCallee)
   assert finalCallee.ast != nil
-  if x.hasFauxMatch:
+  if x.matchedErrorType:
     result = x.call
     result[0] = newSymNode(finalCallee, getCallLineInfo(result[0]))
-    if containsGenericType(result.typ) or x.fauxMatch == tyUnknown:
-      result.typ = newTypeS(x.fauxMatch, c)
-      if result.typ.kind == tyError: incl result.typ.flags, tfCheckedForDestructor
+    if containsGenericType(result.typ):
+      result.typ() = newTypeS(tyError, c)
+      incl result.typ.flags, tfCheckedForDestructor
     return
   let gp = finalCallee.ast[genericParamsPos]
   if gp.isGenericParams:
@@ -717,16 +872,18 @@ proc semResolvedCall(c: PContext, x: var TCandidate,
           # this node will be used in template substitution,
           # pretend this is an untyped node and let regular sem handle the type
           # to prevent problems where a generic parameter is treated as a value
-          tn.typ = nil
+          tn.typ() = nil
           x.call.add tn
         else:
           internalAssert c.config, false
 
   result = x.call
   instGenericConvertersSons(c, result, x)
+  markConvertersUsed(c, result)
   result[0] = newSymNode(finalCallee, getCallLineInfo(result[0]))
-  result.typ = finalCallee.typ.returnType
-  updateDefaultParams(result)
+  if finalCallee.magic notin {mArrGet, mArrPut}:
+    result.typ() = finalCallee.typ.returnType
+  updateDefaultParams(c, result)
 
 proc canDeref(n: PNode): bool {.inline.} =
   result = n.len >= 2 and (let t = n[1].typ;
@@ -734,7 +891,7 @@ proc canDeref(n: PNode): bool {.inline.} =
 
 proc tryDeref(n: PNode): PNode =
   result = newNodeI(nkHiddenDeref, n.info)
-  result.typ = n.typ.skipTypes(abstractInst)[0]
+  result.typ() = n.typ.skipTypes(abstractInst)[0]
   result.add n
 
 proc semOverloadedCall(c: PContext, n, nOrig: PNode,
@@ -751,9 +908,9 @@ proc semOverloadedCall(c: PContext, n, nOrig: PNode,
               candidates)
     result = semResolvedCall(c, r, n, flags, expectedType)
   else:
-    if efDetermineType in flags and c.inGenericContext > 0 and c.matchedConcept == nil:
+    if c.inGenericContext > 0 and c.matchedConcept == nil:
       result = semGenericStmt(c, n)
-      result.typ = makeTypeFromExpr(c, result.copyTree)
+      result.typ() = makeTypeFromExpr(c, result.copyTree)
     elif efExplain notin flags:
       # repeat the overload resolution,
       # this time enabling all the diagnostic output (this should fail again)
@@ -768,22 +925,22 @@ proc explicitGenericInstError(c: PContext; n: PNode): PNode =
   localError(c.config, getCallLineInfo(n), errCannotInstantiateX % renderTree(n))
   result = n
 
-proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode =
+proc explicitGenericSym(c: PContext, n: PNode, s: PSym, errors: var CandidateErrors, doError: bool): PNode =
+  if s.kind in {skTemplate, skMacro}:
+    internalError c.config, n.info, "cannot get explicitly instantiated symbol of " &
+      (if s.kind == skTemplate: "template" else: "macro")
   # binding has to stay 'nil' for this to work!
   var m = newCandidate(c, s, nil)
-
-  for i in 1..<n.len:
-    let formal = s.ast[genericParamsPos][i-1].typ
-    var arg = n[i].typ
-    # try transforming the argument into a static one before feeding it into
-    # typeRel
-    if formal.kind == tyStatic and arg.kind != tyStatic:
-      let evaluated = c.semTryConstExpr(c, n[i], n[i].typ)
-      if evaluated != nil:
-        arg = newTypeS(tyStatic, c, son = evaluated.typ)
-        arg.n = evaluated
-    let tm = typeRel(m, formal, arg)
-    if tm in {isNone, isConvertible}: return nil
+  matchGenericParams(m, n, s)
+  if m.state != csMatch:
+    # state is csMatch only if *all* generic params were matched,
+    # including implicit parameters
+    if doError:
+      errors.add(CandidateError(
+        sym: s,
+        firstMismatch: m.firstMismatch,
+        diagnostics: m.diagnostics))
+    return nil
   var newInst = generateInstance(c, s, m.bindings, n.info)
   newInst.typ.flags.excl tfUnresolved
   let info = getCallLineInfo(n)
@@ -802,46 +959,43 @@ proc setGenericParams(c: PContext, n, expectedParams: PNode) =
           nil
       e = semExprWithType(c, n[i], expectedType = constraint)
     if e.typ == nil:
-      n[i].typ = errorType(c)
+      n[i].typ() = errorType(c)
     else:
-      n[i].typ = e.typ.skipTypes({tyTypeDesc})
+      n[i].typ() = e.typ.skipTypes({tyTypeDesc})
 
-proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
+proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym, doError: bool): PNode =
   assert n.kind == nkBracketExpr
   setGenericParams(c, n, s.ast[genericParamsPos])
   var s = s
   var a = n[0]
+  var errors: CandidateErrors = @[]
   if a.kind == nkSym:
     # common case; check the only candidate has the right
     # number of generic type parameters:
-    if s.ast[genericParamsPos].safeLen != n.len-1:
-      let expected = s.ast[genericParamsPos].safeLen
-      localError(c.config, getCallLineInfo(n), errGenerated, "cannot instantiate: '" & renderTree(n) &
-         "'; got " & $(n.len-1) & " typeof(s) but expected " & $expected)
-      return n
-    result = explicitGenericSym(c, n, s)
-    if result == nil: result = explicitGenericInstError(c, n)
+    result = explicitGenericSym(c, n, s, errors, doError)
+    if doError and result == nil:
+      notFoundError(c, n, errors)
   elif a.kind in {nkClosedSymChoice, nkOpenSymChoice}:
     # choose the generic proc with the proper number of type parameters.
-    # XXX I think this could be improved by reusing sigmatch.paramTypesMatch.
-    # It's good enough for now.
     result = newNodeI(a.kind, getCallLineInfo(n))
     for i in 0..<a.len:
       var candidate = a[i].sym
       if candidate.kind in {skProc, skMethod, skConverter,
                             skFunc, skIterator}:
-        # it suffices that the candidate has the proper number of generic
-        # type parameters:
-        if candidate.ast[genericParamsPos].safeLen == n.len-1:
-          let x = explicitGenericSym(c, n, candidate)
-          if x != nil: result.add(x)
+        let x = explicitGenericSym(c, n, candidate, errors, doError)
+        if x != nil: result.add(x)
     # get rid of nkClosedSymChoice if not ambiguous:
-    if result.len == 1 and a.kind == nkClosedSymChoice:
-      result = result[0]
-    elif result.len == 0: result = explicitGenericInstError(c, n)
-    # candidateCount != 1: return explicitGenericInstError(c, n)
+    if result.len == 0:
+      result = nil
+      if doError:
+        notFoundError(c, n, errors)
   else:
-    result = explicitGenericInstError(c, n)
+    # probably unreachable: we are trying to instantiate `a` which is not
+    # a sym/symchoice
+    if doError:
+      result = explicitGenericInstError(c, n)
+    else:
+      result = nil
 
 proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): tuple[s: PSym, state: TBorrowState] =
   # Searches for the fn in the symbol table. If the parameter lists are suitable

compiler/semdata.nim

@@ -15,8 +15,8 @@ when defined(nimPreviewSlimSystem):
   import std/assertions
 
 import
-  options, ast, astalgo, msgs, idents, renderer,
-  magicsys, vmdef, modulegraphs, lineinfos, pathutils
+  options, ast, msgs, idents, renderer,
+  magicsys, vmdef, modulegraphs, lineinfos, pathutils, layeredtable
 
 import ic / ic
 
@@ -73,9 +73,9 @@ type
     efNoUndeclared, efIsDotCall, efCannotBeDotCall,
       # Use this if undeclared identifiers should not raise an error during
       # overload resolution.
-    efNoDiagnostics,
     efTypeAllowed # typeAllowed will be called after
     efWantNoDefaults
+    efIgnoreDefaults # var statements without initialization
     efAllowSymChoice # symchoice node should not be resolved
 
   TExprFlags* = set[TExprFlag]
@@ -136,9 +136,13 @@ type
     semOverloadedCall*: proc (c: PContext, n, nOrig: PNode,
                               filter: TSymKinds, flags: TExprFlags, expectedType: PType = nil): PNode {.nimcall.}
     semTypeNode*: proc(c: PContext, n: PNode, prev: PType): PType {.nimcall.}
-    semInferredLambda*: proc(c: PContext, pt: Table[ItemId, PType], n: PNode): PNode
-    semGenerateInstance*: proc (c: PContext, fn: PSym, pt: Table[ItemId, PType],
+    semInferredLambda*: proc(c: PContext, pt: LayeredIdTable, n: PNode): PNode
+    semGenerateInstance*: proc (c: PContext, fn: PSym, pt: LayeredIdTable,
                                 info: TLineInfo): PSym
+    instantiateOnlyProcType*: proc (c: PContext, pt: LayeredIdTable,
+                                    prc: PSym, info: TLineInfo): PType
+      # used by sigmatch for explicit generic instantiations
+    fitDefaultNode*: proc (c: PContext, n: var PNode, expectedType: PType)
     includedFiles*: IntSet    # used to detect recursive include files
     pureEnumFields*: TStrTable   # pure enum fields that can be used unambiguously
     userPragmas*: TStrTable
@@ -168,6 +172,7 @@ type
     inUncheckedAssignSection*: int
     importModuleLookup*: Table[int, seq[int]] # (module.ident.id, [module.id])
     skipTypes*: seq[PNode] # used to skip types between passes in type section. So far only used for inheritance, sets and generic bodies.
+    inTypeofContext*: int
   TBorrowState* = enum
     bsNone, bsReturnNotMatch, bsNoDistinct, bsGeneric, bsNotSupported, bsMatch
 
@@ -191,29 +196,29 @@ proc getIntLitType*(c: PContext; literal: PNode): PType =
 proc setIntLitType*(c: PContext; result: PNode) =
   let i = result.intVal
   case c.config.target.intSize
-  of 8: result.typ = getIntLitType(c, result)
+  of 8: result.typ() = getIntLitType(c, result)
   of 4:
     if i >= low(int32) and i <= high(int32):
-      result.typ = getIntLitType(c, result)
+      result.typ() = getIntLitType(c, result)
     else:
-      result.typ = getSysType(c.graph, result.info, tyInt64)
+      result.typ() = getSysType(c.graph, result.info, tyInt64)
   of 2:
     if i >= low(int16) and i <= high(int16):
-      result.typ = getIntLitType(c, result)
+      result.typ() = getIntLitType(c, result)
     elif i >= low(int32) and i <= high(int32):
-      result.typ = getSysType(c.graph, result.info, tyInt32)
+      result.typ() = getSysType(c.graph, result.info, tyInt32)
     else:
-      result.typ = getSysType(c.graph, result.info, tyInt64)
+      result.typ() = getSysType(c.graph, result.info, tyInt64)
   of 1:
     # 8 bit CPUs are insane ...
     if i >= low(int8) and i <= high(int8):
-      result.typ = getIntLitType(c, result)
+      result.typ() = getIntLitType(c, result)
     elif i >= low(int16) and i <= high(int16):
-      result.typ = getSysType(c.graph, result.info, tyInt16)
+      result.typ() = getSysType(c.graph, result.info, tyInt16)
     elif i >= low(int32) and i <= high(int32):
-      result.typ = getSysType(c.graph, result.info, tyInt32)
+      result.typ() = getSysType(c.graph, result.info, tyInt32)
     else:
-      result.typ = getSysType(c.graph, result.info, tyInt64)
+      result.typ() = getSysType(c.graph, result.info, tyInt64)
   else:
     internalError(c.config, result.info, "invalid int size")
 
@@ -445,7 +450,7 @@ when false:
 proc makeStaticExpr*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkStaticExpr, n.info)
   result.sons = @[n]
-  result.typ = if n.typ != nil and n.typ.kind == tyStatic: n.typ
+  result.typ() = if n.typ != nil and n.typ.kind == tyStatic: n.typ
                else: newTypeS(tyStatic, c, n.typ)
 
 proc makeAndType*(c: PContext, t1, t2: PType): PType =
@@ -504,7 +509,7 @@ proc errorType*(c: PContext): PType =
 
 proc errorNode*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkEmpty, n.info)
-  result.typ = errorType(c)
+  result.typ() = errorType(c)
 
 # These mimic localError
 template localErrorNode*(c: PContext, n: PNode, info: TLineInfo, msg: TMsgKind, arg: string): PNode =
@@ -525,10 +530,11 @@ template localErrorNode*(c: PContext, n: PNode, arg: string): PNode =
   liMessage(c.config, n2.info, errGenerated, arg, doNothing, instLoc())
   errorNode(c, n2)
 
-proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) =
-  dest.kind = kind
-  dest.owner = getCurrOwner(c)
-  dest.size = - 1
+when false:
+  proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) =
+    dest.kind = kind
+    dest.owner = getCurrOwner(c)
+    dest.size = - 1
 
 proc makeRangeType*(c: PContext; first, last: BiggestInt;
                     info: TLineInfo; intType: PType = nil): PType =
@@ -559,7 +565,7 @@ proc symFromType*(c: PContext; t: PType, info: TLineInfo): PSym =
 
 proc symNodeFromType*(c: PContext, t: PType, info: TLineInfo): PNode =
   result = newSymNode(symFromType(c, t, info), info)
-  result.typ = makeTypeDesc(c, t)
+  result.typ() = makeTypeDesc(c, t)
 
 proc markIndirect*(c: PContext, s: PSym) {.inline.} =
   if s.kind in {skProc, skFunc, skConverter, skMethod, skIterator}:

compiler/semfields.nim

@@ -18,6 +18,15 @@ type
     replaceByFieldName: bool
     c: PContext
 
+proc wrapNewScope(c: PContext, n: PNode): PNode {.inline.} =
+  # use `if true` to not interfere with `break`
+  # just opening scope via `openScope(c)` isn't enough,
+  # a scope has to be opened in the codegen as well for reused
+  # template instantiations
+  let trueLit = newIntLit(c.graph, n.info, 1)
+  trueLit.typ() = getSysType(c.graph, n.info, tyBool)
+  result = newTreeI(nkIfStmt, n.info, newTreeI(nkElifBranch, n.info, trueLit, n))
+
 proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode =
   if c.field != nil and isEmptyType(c.field.typ):
     result = newNode(nkEmpty)
@@ -72,7 +81,9 @@ proc semForObjectFields(c: TFieldsCtx, typ, forLoop, father: PNode) =
     )
     openScope(c.c)
     inc c.c.inUnrolledContext
-    let body = instFieldLoopBody(fc, lastSon(forLoop), forLoop)
+    var body = instFieldLoopBody(fc, lastSon(forLoop), forLoop)
+    # new scope for each field that codegen should know about:
+    body = wrapNewScope(c.c, body)
     father.add(semStmt(c.c, body, {}))
     dec c.c.inUnrolledContext
     closeScope(c.c)
@@ -148,6 +159,8 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode =
           replaceByFieldName: m == mFieldPairs
       )
       var body = instFieldLoopBody(fc, loopBody, n)
+      # new scope for each field that codegen should know about:
+      body = wrapNewScope(c, body)
       inc c.inUnrolledContext
       stmts.add(semStmt(c, body, {}))
       dec c.inUnrolledContext

compiler/semfold.nim

@@ -38,7 +38,7 @@ proc newIntNodeT*(intVal: Int128, n: PNode; idgen: IdGenerator; g: ModuleGraph):
   # original type was 'int', not a distinct int etc.
   if n.typ.kind == tyInt:
     # access cache for the int lit type
-    result.typ = getIntLitTypeG(g, result, idgen)
+    result.typ() = getIntLitTypeG(g, result, idgen)
   result.info = n.info
 
 proc newFloatNodeT*(floatVal: BiggestFloat, n: PNode; g: ModuleGraph): PNode =
@@ -46,12 +46,12 @@ proc newFloatNodeT*(floatVal: BiggestFloat, n: PNode; g: ModuleGraph): PNode =
     result = newFloatNode(nkFloat32Lit, floatVal)
   else:
     result = newFloatNode(nkFloatLit, floatVal)
-  result.typ = n.typ
+  result.typ() = n.typ
   result.info = n.info
 
 proc newStrNodeT*(strVal: string, n: PNode; g: ModuleGraph): PNode =
   result = newStrNode(nkStrLit, strVal)
-  result.typ = n.typ
+  result.typ() = n.typ
   result.info = n.info
 
 proc getConstExpr*(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
@@ -302,6 +302,9 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P
   of mMinusSet:
     result = nimsets.diffSets(g.config, a, b)
     result.info = n.info
+  of mXorSet:
+    result = nimsets.symdiffSets(g.config, a, b)
+    result.info = n.info
   of mConStrStr: result = newStrNodeT(getStrOrChar(a) & getStrOrChar(b), n, g)
   of mInSet: result = newIntNodeT(toInt128(ord(inSet(a, b))), n, idgen, g)
   of mRepr:
@@ -316,7 +319,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P
   of mEnumToStr: result = newStrNodeT(ordinalValToString(a, g), n, g)
   of mArrToSeq:
     result = copyTree(a)
-    result.typ = n.typ
+    result.typ() = n.typ
   of mCompileOption:
     result = newIntNodeT(toInt128(ord(commands.testCompileOption(g.config, a.getStr, n.info))), n, idgen, g)
   of mCompileOptionArg:
@@ -411,7 +414,7 @@ proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): P
       result = newIntNodeT(toInt128(a.getOrdValue != 0), n, idgen, g)
     of tyBool, tyEnum: # xxx shouldn't we disallow `tyEnum`?
       result = a
-      result.typ = n.typ
+      result.typ() = n.typ
     else:
       raiseAssert $srcTyp.kind
   of tyInt..tyInt64, tyUInt..tyUInt64:
@@ -420,14 +423,17 @@ proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): P
       result = newIntNodeT(toInt128(getFloat(a)), n, idgen, g)
     of tyChar, tyUInt..tyUInt64, tyInt..tyInt64:
       var val = a.getOrdValue
-      if check: rangeCheck(n, val, g)
-      result = newIntNodeT(val, n, idgen, g)
       if dstTyp.kind in {tyUInt..tyUInt64}:
+        result = newIntNodeT(maskBytes(val, int getSize(g.config, dstTyp)), n, idgen, g)
         result.transitionIntKind(nkUIntLit)
+      else:
+        if check: rangeCheck(n, val, g)
+        result = newIntNodeT(val, n, idgen, g)
     else:
       result = a
-      result.typ = n.typ
-    if check and result.kind in {nkCharLit..nkUInt64Lit}:
+      result.typ() = n.typ
+    if check and result.kind in {nkCharLit..nkUInt64Lit} and
+          dstTyp.kind notin {tyUInt..tyUInt64}:
       rangeCheck(n, getInt(result), g)
   of tyFloat..tyFloat64:
     case srcTyp.kind
@@ -435,12 +441,12 @@ proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): P
       result = newFloatNodeT(toFloat64(getOrdValue(a)), n, g)
     else:
       result = a
-      result.typ = n.typ
+      result.typ() = n.typ
   of tyOpenArray, tyVarargs, tyProc, tyPointer:
     result = nil
   else:
     result = a
-    result.typ = n.typ
+    result.typ() = n.typ
 
 proc getArrayConstr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode =
   if n.kind == nkBracket:
@@ -511,10 +517,10 @@ proc foldConStrStr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
 proc newSymNodeTypeDesc*(s: PSym; idgen: IdGenerator; info: TLineInfo): PNode =
   result = newSymNode(s, info)
   if s.typ.kind != tyTypeDesc:
-    result.typ = newType(tyTypeDesc, idgen, s.owner)
+    result.typ() = newType(tyTypeDesc, idgen, s.owner)
     result.typ.addSonSkipIntLit(s.typ, idgen)
   else:
-    result.typ = s.typ
+    result.typ() = s.typ
 
 proc foldDefine(m, s: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode =
   result = nil
@@ -633,7 +639,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
       if s.typ.kind == tyStatic:
         if s.typ.n != nil and tfUnresolved notin s.typ.flags:
           result = s.typ.n
-          result.typ = s.typ.base
+          result.typ() = s.typ.base
       elif s.typ.isIntLit:
         result = s.typ.n
       else:
@@ -701,10 +707,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
     except DivByZeroDefect:
       localError(g.config, n.info, "division by zero")
   of nkAddr:
-    var a = getConstExpr(m, n[0], idgen, g)
-    if a != nil:
-      result = n
-      n[0] = a
+    result = nil # don't fold paths containing nkAddr
   of nkBracket, nkCurly:
     result = copyNode(n)
     for son in n.items:
@@ -749,7 +752,9 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
     if a == nil: return
     if leValueConv(n[1], a) and leValueConv(a, n[2]):
       result = a              # a <= x and x <= b
-      result.typ = n.typ
+      result.typ() = n.typ
+    elif n.typ.kind in {tyUInt..tyUInt64}:
+      discard "don't check uints"
     else:
       localError(g.config, n.info,
         "conversion from $1 to $2 is invalid" %
@@ -758,7 +763,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
     var a = getConstExpr(m, n[0], idgen, g)
     if a == nil: return
     result = a
-    result.typ = n.typ
+    result.typ() = n.typ
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
     var a = getConstExpr(m, n[1], idgen, g)
     if a == nil: return
@@ -771,10 +776,11 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
   of nkCast:
     var a = getConstExpr(m, n[1], idgen, g)
     if a == nil: return
-    if n.typ != nil and n.typ.kind in NilableTypes:
+    if n.typ != nil and n.typ.kind in NilableTypes and
+        not (n.typ.kind == tyProc and a.typ.kind == tyProc):
       # we allow compile-time 'cast' for pointer types:
       result = a
-      result.typ = n.typ
+      result.typ() = n.typ
   of nkBracketExpr: result = foldArrayAccess(m, n, idgen, g)
   of nkDotExpr: result = foldFieldAccess(m, n, idgen, g)
   of nkCheckedFieldExpr:

compiler/semgnrc.nim

@@ -61,6 +61,7 @@ template canOpenSym(s): bool =
 
 proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
                           ctx: var GenericCtx; flags: TSemGenericFlags,
+                          isAmbiguous: bool,
                           fromDotExpr=false): PNode =
   result = nil
   semIdeForTemplateOrGenericCheck(c.config, n, ctx.cursorInBody)
@@ -72,9 +73,15 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
         result.transitionSonsKind(nkClosedSymChoice)
     else:
       result = symChoice(c, n, s, scOpen)
-      if result.kind == nkSym and canOpenSym(result.sym):
-        result.flags.incl nfOpenSym
-        result.typ = nil
+      if canOpenSym(s):
+        if openSym in c.features:
+          if result.kind == nkSym:
+            result = newOpenSym(result)
+          else:
+            result.typ() = nil
+        else:
+          result.flags.incl nfDisabledOpenSym
+          result.typ() = nil
   case s.kind
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
@@ -98,13 +105,28 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
     if s.typ != nil and s.typ.kind == tyStatic:
       if s.typ.n != nil:
         result = s.typ.n
+      elif c.inGenericContext > 0 and withinConcept notin flags:
+        # don't leave generic param as identifier node in generic type,
+        # sigmatch will try to instantiate generic type AST without all params
+        # fine to give a symbol node a generic type here since
+        # we are in a generic context and `prepareNode` will be called
+        result = newSymNodeTypeDesc(s, c.idgen, n.info)
+        if canOpenSym(result.sym):
+          if openSym in c.features:
+            result = newOpenSym(result)
+          else:
+            result.flags.incl nfDisabledOpenSym
+            result.typ() = nil
       else:
         result = n
     else:
       result = newSymNodeTypeDesc(s, c.idgen, n.info)
       if canOpenSym(result.sym):
-        result.flags.incl nfOpenSym
-        result.typ = nil
+        if openSym in c.features:
+          result = newOpenSym(result)
+        else:
+          result.flags.incl nfDisabledOpenSym
+          result.typ() = nil
     onUse(n.info, s)
   of skParam:
     result = n
@@ -112,18 +134,41 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
   of skType:
     if (s.typ != nil) and
        (s.typ.flags * {tfGenericTypeParam, tfImplicitTypeParam} == {}):
+      if isAmbiguous:
+        # ambiguous types should be symchoices since lookup behaves
+        # differently for them in regular expressions
+        maybeDotChoice(c, n, s, fromDotExpr)
+        return
       result = newSymNodeTypeDesc(s, c.idgen, n.info)
       if canOpenSym(result.sym):
-        result.flags.incl nfOpenSym
-        result.typ = nil
+        if openSym in c.features:
+          result = newOpenSym(result)
+        else:
+          result.flags.incl nfDisabledOpenSym
+          result.typ() = nil
+    elif c.inGenericContext > 0 and withinConcept notin flags:
+      # don't leave generic param as identifier node in generic type,
+      # sigmatch will try to instantiate generic type AST without all params
+      # fine to give a symbol node a generic type here since
+      # we are in a generic context and `prepareNode` will be called
+      result = newSymNodeTypeDesc(s, c.idgen, n.info)
+      if canOpenSym(result.sym):
+        if openSym in c.features:
+          result = newOpenSym(result)
+        else:
+          result.flags.incl nfDisabledOpenSym
+          result.typ() = nil
     else:
       result = n
     onUse(n.info, s)
   else:
     result = newSymNode(s, n.info)
     if canOpenSym(result.sym):
-      result.flags.incl nfOpenSym
-      result.typ = nil
+      if openSym in c.features:
+        result = newOpenSym(result)
+      else:
+        result.flags.incl nfDisabledOpenSym
+        result.typ() = nil
     onUse(n.info, s)
 
 proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
@@ -145,7 +190,7 @@ proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
     elif s.isMixedIn:
       result = symChoice(c, n, s, scForceOpen)
     else:
-      result = semGenericStmtSymbol(c, n, s, ctx, flags)
+      result = semGenericStmtSymbol(c, n, s, ctx, flags, amb)
   # else: leave as nkIdent
 
 proc newDot(n, b: PNode): PNode =
@@ -161,10 +206,11 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
 
   let luf = if withinMixin notin flags: {checkUndeclared, checkModule} else: {checkModule}
 
+  c.isAmbiguous = false
   var s = qualifiedLookUp(c, n, luf)
   if s != nil:
     isMacro = s.kind in {skTemplate, skMacro}
-    result = semGenericStmtSymbol(c, n, s, ctx, flags)
+    result = semGenericStmtSymbol(c, n, s, ctx, flags, c.isAmbiguous)
   else:
     n[0] = semGenericStmt(c, n[0], flags, ctx)
     result = n
@@ -178,24 +224,21 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
       isMacro = s.kind in {skTemplate, skMacro}
       if withinBind in flags or s.id in ctx.toBind:
         if s.kind == skType: # don't put types in sym choice
-          result = newDot(result, semGenericStmtSymbol(c, n, s, ctx, flags, fromDotExpr=true))
+          var ambig = false
+          if candidates.len > 1:
+            let s2 = searchInScopes(c, ident, ambig)
+          result = newDot(result, semGenericStmtSymbol(c, n, s, ctx, flags,
+            isAmbiguous = ambig, fromDotExpr = true))
         else:
           result = newDot(result, symChoice(c, n, s, scClosed))
       elif s.isMixedIn:
         result = newDot(result, symChoice(c, n, s, scForceOpen))
       else:
+        var ambig = false
         if s.kind == skType and candidates.len > 1:
-          var ambig = false
-          let s2 = searchInScopes(c, ident, ambig) 
-          if ambig:
-            # this is a type conversion like a.T where T is ambiguous with
-            # other types or routines
-            # in regular code, this never considers a type conversion and
-            # skips to routine overloading
-            # so symchoices are used which behave similarly with type symbols
-            result = newDot(result, symChoice(c, n, s, scForceOpen))
-            return
-        let syms = semGenericStmtSymbol(c, n, s, ctx, flags, fromDotExpr=true)
+          discard searchInScopes(c, ident, ambig)
+        let syms = semGenericStmtSymbol(c, n, s, ctx, flags,
+          isAmbiguous = ambig, fromDotExpr = true)
         result = newDot(result, syms)
 
 proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) =
@@ -262,7 +305,9 @@ proc semGenericStmt(c: PContext, n: PNode,
     # check if it is an expression macro:
     checkMinSonsLen(n, 1, c.config)
     let fn = n[0]
+    c.isAmbiguous = false
     var s = qualifiedLookUp(c, fn, {})
+    let ambig = c.isAmbiguous
     if s == nil and
         {withinMixin, withinConcept}*flags == {} and
         fn.kind in {nkIdent, nkAccQuoted} and
@@ -315,7 +360,12 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skType:
         # bad hack for generics:
         if (s.typ != nil) and (s.typ.kind != tyGenericParam):
-          result[0] = newSymNodeTypeDesc(s, c.idgen, fn.info)
+          if ambig:
+            # ambiguous types should be symchoices since lookup behaves
+            # differently for them in regular expressions
+            result[0] = sc
+          else:
+            result[0] = newSymNodeTypeDesc(s, c.idgen, fn.info)
           onUse(fn.info, s)
           first = 1
       else:
@@ -563,9 +613,27 @@ proc semGenericStmt(c: PContext, n: PNode,
       else:
         body = getBody(c.graph, s)
     else: body = n[bodyPos]
-    n[bodyPos] = semGenericStmtScope(c, body, flags, ctx)
+    let bodyFlags = if n.kind == nkTemplateDef: flags + {withinMixin} else: flags
+    n[bodyPos] = semGenericStmtScope(c, body, bodyFlags, ctx)
     closeScope(c)
-  of nkPragma, nkPragmaExpr: discard
+  of nkPragmaExpr:
+    result[1] = semGenericStmt(c, n[1], flags, ctx)
+  of nkPragma:
+    for i in 0 ..< n.len:
+      let x = n[i]
+      let prag = whichPragma(x)
+      if x.kind in nkPragmaCallKinds:
+        # process each child individually to prevent untyped macros/templates
+        # from instantiating
+        # if pragma is language-level pragma, skip name node:
+        let start = ord(prag != wInvalid)
+        for j in start ..< x.len:
+          # treat as mixin context for user pragmas & macro args
+          x[j] = semGenericStmt(c, x[j], flags+{withinMixin}, ctx)
+      elif prag == wInvalid:
+        # only sem if not a language-level pragma 
+        # treat as mixin context for user pragmas & macro args
+        result[i] = semGenericStmt(c, x, flags+{withinMixin}, ctx)
   of nkExprColonExpr, nkExprEqExpr:
     checkMinSonsLen(n, 2, c.config)
     result[1] = semGenericStmt(c, n[1], flags, ctx)

compiler/seminst.nim

@@ -34,7 +34,7 @@ proc pushProcCon*(c: PContext; owner: PSym) =
 const
   errCannotInstantiateX = "cannot instantiate: '$1'"
 
-iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TypeMapping): PSym =
+iterator instantiateGenericParamList(c: PContext, n: PNode, pt: LayeredIdTable): PSym =
   internalAssert c.config, n.kind == nkGenericParams
   for a in n.items:
     internalAssert c.config, a.kind == nkSym
@@ -43,7 +43,7 @@ iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TypeMapping): PS
       let symKind = if q.typ.kind == tyStatic: skConst else: skType
       var s = newSym(symKind, q.name, c.idgen, getCurrOwner(c), q.info)
       s.flags.incl {sfUsed, sfFromGeneric}
-      var t = idTableGet(pt, q.typ)
+      var t = lookup(pt, q.typ)
       if t == nil:
         if tfRetType in q.typ.flags:
           # keep the generic type and allow the return type to be bound
@@ -53,11 +53,16 @@ iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TypeMapping): PS
           if q.typ.kind != tyCompositeTypeClass:
             localError(c.config, a.info, errCannotInstantiateX % s.name.s)
           t = errorType(c)
-      elif t.kind in {tyGenericParam, tyConcept}:
+      elif t.kind in {tyGenericParam, tyConcept, tyFromExpr} or
+          # generic body types are accepted as typedesc arguments
+          (t.kind == tyGenericBody and q.typ.kind != tyTypeDesc):
         localError(c.config, a.info, errCannotInstantiateX % q.name.s)
         t = errorType(c)
-      elif isUnresolvedStatic(t) and c.inGenericContext == 0 and
-          c.matchedConcept == nil:
+      elif isUnresolvedStatic(t) and (q.typ.kind == tyStatic or
+            (q.typ.kind == tyGenericParam and
+              q.typ.genericParamHasConstraints and
+              q.typ.genericConstraint.kind == tyStatic)) and
+          c.inGenericContext == 0 and c.matchedConcept == nil:
         # generic/concept type bodies will try to instantiate static values but
         # won't actually use them
         localError(c.config, a.info, errCannotInstantiateX % q.name.s)
@@ -106,7 +111,7 @@ proc freshGenSyms(c: PContext; n: PNode, owner, orig: PSym, symMap: var SymMappi
     elif s.owner == nil or s.owner.kind == skPackage:
       #echo "copied this ", s.name.s
       x = copySym(s, c.idgen)
-      x.owner = owner
+      setOwner(x, owner)
       idTablePut(symMap, s, x)
       n.sym = x
   else:
@@ -217,7 +222,7 @@ proc referencesAnotherParam(n: PNode, p: PSym): bool =
       if referencesAnotherParam(n[i], p): return true
     return false
 
-proc instantiateProcType(c: PContext, pt: TypeMapping,
+proc instantiateProcType(c: PContext, pt: LayeredIdTable,
                          prc: PSym, info: TLineInfo) =
   # XXX: Instantiates a generic proc signature, while at the same
   # time adding the instantiated proc params into the current scope.
@@ -234,7 +239,7 @@ proc instantiateProcType(c: PContext, pt: TypeMapping,
   # will need to use openScope, addDecl, etc.
   #addDecl(c, prc)
   pushInfoContext(c.config, info)
-  var typeMap = initLayeredTypeMap(pt)
+  var typeMap = shallowCopy(pt) # use previous bindings without writing to them
   var cl = initTypeVars(c, typeMap, info, nil)
   var result = instCopyType(cl, prc.typ)
   let originalParams = result.n
@@ -250,9 +255,15 @@ proc instantiateProcType(c: PContext, pt: TypeMapping,
     var typeToFit = resulti
 
     let needsStaticSkipping = resulti.kind == tyFromExpr
+    let needsTypeDescSkipping = resulti.kind == tyTypeDesc and tfUnresolved in resulti.flags
+    if resulti.kind == tyFromExpr:
+      resulti.flags.incl tfNonConstExpr
     result[i] = replaceTypeVarsT(cl, resulti)
     if needsStaticSkipping:
       result[i] = result[i].skipTypes({tyStatic})
+    if needsTypeDescSkipping:
+      result[i] = result[i].skipTypes({tyTypeDesc})
+      typeToFit = result[i]
 
     # ...otherwise, we use the instantiated type in `fitNode`
     if (typeToFit.kind != tyTypeDesc or typeToFit.base.kind != tyNone) and
@@ -262,7 +273,7 @@ proc instantiateProcType(c: PContext, pt: TypeMapping,
     internalAssert c.config, originalParams[i].kind == nkSym
     let oldParam = originalParams[i].sym
     let param = copySym(oldParam, c.idgen)
-    param.owner = prc
+    setOwner(param, prc)
     param.typ = result[i]
 
     # The default value is instantiated and fitted against the final
@@ -270,9 +281,10 @@ proc instantiateProcType(c: PContext, pt: TypeMapping,
     # call head symbol, because this leads to infinite recursion.
     if oldParam.ast != nil:
       var def = oldParam.ast.copyTree
-      if def.kind in nkCallKinds:
-        for i in 1..<def.len:
-          def[i] = replaceTypeVarsN(cl, def[i], 1)
+      if def.typ.kind == tyFromExpr:
+        def.typ.flags.incl tfNonConstExpr
+      if not isIntLit(def.typ):
+        def = prepareNode(cl, def)
 
       # allow symchoice since node will be fit later
       # although expectedType should cover it
@@ -289,6 +301,8 @@ proc instantiateProcType(c: PContext, pt: TypeMapping,
         # the user calls an explicit instantiation of the proc (this is
         # the only way the default value might be inserted).
         param.ast = errorNode(c, def)
+        # we know the node is empty, we need the actual type for error message
+        param.ast.typ() = def.typ
       else:
         param.ast = fitNodePostMatch(c, typeToFit, converted)
       param.typ = result[i]
@@ -312,6 +326,22 @@ proc instantiateProcType(c: PContext, pt: TypeMapping,
   prc.typ = result
   popInfoContext(c.config)
 
+proc instantiateOnlyProcType(c: PContext, pt: LayeredIdTable, prc: PSym, info: TLineInfo): PType =
+  # instantiates only the type of a given proc symbol
+  # used by sigmatch for explicit generics
+  # wouldn't be needed if sigmatch could handle complex cases,
+  # examples are in texplicitgenerics
+  # might be buggy, see rest of generateInstance if problems occur
+  let fakeSym = copySym(prc, c.idgen)
+  incl(fakeSym.flags, sfFromGeneric)
+  fakeSym.instantiatedFrom = prc
+  openScope(c)
+  for s in instantiateGenericParamList(c, prc.ast[genericParamsPos], pt):
+    addDecl(c, s)
+  instantiateProcType(c, pt, fakeSym, info)
+  closeScope(c)
+  result = fakeSym.typ
+
 proc fillMixinScope(c: PContext) =
   var p = c.p
   while p != nil:
@@ -332,7 +362,7 @@ proc getLocalPassC(c: PContext, s: PSym): string =
     for p in n:
       extractPassc(p)
 
-proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
+proc generateInstance(c: PContext, fn: PSym, pt: LayeredIdTable,
                       info: TLineInfo): PSym =
   ## Generates a new instance of a generic procedure.
   ## The `pt` parameter is a type-unsafe mapping table used to link generic
@@ -342,7 +372,12 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
   # generates an instantiated proc
   if c.instCounter > 50:
     globalError(c.config, info, "generic instantiation too nested")
-  inc(c.instCounter)
+  inc c.instCounter
+  let currentTypeofContext = c.inTypeofContext
+  c.inTypeofContext = 0
+  defer:
+    dec c.instCounter
+    c.inTypeofContext = currentTypeofContext
   # careful! we copy the whole AST including the possibly nil body!
   var n = copyTree(fn.ast)
   # NOTE: for access of private fields within generics from a different module
@@ -360,9 +395,9 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
     let passc = getLocalPassC(c, producer)
     if passc != "": #pass the local compiler options to the consumer module too
       extccomp.addLocalCompileOption(c.config, passc, toFullPathConsiderDirty(c.config, c.module.info.fileIndex))
-    result.owner = c.module
+    setOwner(result, c.module)
   else:
-    result.owner = fn
+    setOwner(result, fn)
   result.ast = n
   pushOwner(c, result)
 
@@ -396,6 +431,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
   for _, param in paramTypes(result.typ):
     entry.concreteTypes[i] = param
     inc i
+  #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " ", entry.concreteTypes.len
   if tfTriggersCompileTime in result.typ.flags:
     incl(result.flags, sfCompileTime)
   n[genericParamsPos] = c.graph.emptyNode
@@ -424,7 +460,9 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
     if result.magic notin {mSlice, mTypeOf}:
       # 'toOpenArray' is special and it is allowed to return 'openArray':
       paramsTypeCheck(c, result.typ)
+    #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " <-- NEW PROC!", " ", entry.concreteTypes.len
   else:
+    #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " <-- CACHED! ", typeToString(oldPrc.typ), " ", entry.concreteTypes.len
     result = oldPrc
   popProcCon(c)
   popInfoContext(c.config)
@@ -433,7 +471,6 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
   popOwner(c)
   c.currentScope = oldScope
   discard c.friendModules.pop()
-  dec(c.instCounter)
   c.matchedConcept = oldMatchedConcept
   if result.kind == skMethod: finishMethod(c, result)
 

compiler/semmacrosanity.nim

@@ -50,7 +50,8 @@ proc annotateType*(n: PNode, t: PType; conf: ConfigRef) =
   case n.kind
   of nkObjConstr:
     let x = t.skipTypes(abstractPtrs)
-    n.typ = t
+    n.typ() = t
+    n[0].typ() = t
     for i in 1..<n.len:
       var j = i-1
       let field = x.ithField(j)
@@ -61,12 +62,12 @@ proc annotateType*(n: PNode, t: PType; conf: ConfigRef) =
         annotateType(n[i][1], field.typ, conf)
   of nkPar, nkTupleConstr:
     if x.kind == tyTuple:
-      n.typ = t
+      n.typ() = t
       for i in 0..<n.len:
         if i >= x.kidsLen: globalError conf, n.info, "invalid field at index " & $i
         else: annotateType(n[i], x[i], conf)
     elif x.kind == tyProc and x.callConv == ccClosure:
-      n.typ = t
+      n.typ() = t
     elif x.kind == tyOpenArray: # `opcSlice` transforms slices into tuples
       if n.kind == nkTupleConstr:
         let
@@ -87,39 +88,39 @@ proc annotateType*(n: PNode, t: PType; conf: ConfigRef) =
           globalError(conf, n.info, "Incorrectly generated tuple constr")
         n[] = bracketExpr[]
 
-      n.typ = t
+      n.typ() = t
     else:
       globalError(conf, n.info, "() must have a tuple type")
   of nkBracket:
     if x.kind in {tyArray, tySequence, tyOpenArray}:
-      n.typ = t
+      n.typ() = t
       for m in n: annotateType(m, x.elemType, conf)
     else:
       globalError(conf, n.info, "[] must have some form of array type")
   of nkCurly:
     if x.kind in {tySet}:
-      n.typ = t
+      n.typ() = t
       for m in n: annotateType(m, x.elemType, conf)
     else:
       globalError(conf, n.info, "{} must have the set type")
   of nkFloatLit..nkFloat128Lit:
     if x.kind in {tyFloat..tyFloat128}:
-      n.typ = t
+      n.typ() = t
     else:
       globalError(conf, n.info, "float literal must have some float type")
   of nkCharLit..nkUInt64Lit:
     if x.kind in {tyInt..tyUInt64, tyBool, tyChar, tyEnum}:
-      n.typ = t
+      n.typ() = t
     else:
       globalError(conf, n.info, "integer literal must have some int type")
   of nkStrLit..nkTripleStrLit:
     if x.kind in {tyString, tyCstring}:
-      n.typ = t
+      n.typ() = t
     else:
       globalError(conf, n.info, "string literal must be of some string type")
   of nkNilLit:
     if x.kind in NilableTypes+{tyString, tySequence}:
-      n.typ = t
+      n.typ() = t
     else:
       globalError(conf, n.info, "nil literal must be of some pointer type")
   else: discard

compiler/semmagic.nim

@@ -18,7 +18,7 @@ proc addDefaultFieldForNew(c: PContext, n: PNode): PNode =
   let typ = result[1].typ # new(x)
   if typ.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyRef and typ.skipTypes({tyGenericInst, tyAlias, tySink})[0].kind == tyObject:
     var asgnExpr = newTree(nkObjConstr, newNodeIT(nkType, result[1].info, typ))
-    asgnExpr.typ = typ
+    asgnExpr.typ() = typ
     var t = typ.skipTypes({tyGenericInst, tyAlias, tySink})[0]
     while true:
       asgnExpr.sons.add defaultFieldsForTheUninitialized(c, t.n, false)
@@ -38,7 +38,7 @@ proc semAddr(c: PContext; n: PNode): PNode =
   if isAssignable(c, x) notin {arLValue, arLocalLValue, arAddressableConst, arLentValue}:
     localError(c.config, n.info, errExprHasNoAddress)
   result.add x
-  result.typ = makePtrType(c, x.typ)
+  result.typ() = makePtrType(c, x.typ)
 
 proc semTypeOf(c: PContext; n: PNode): PNode =
   var m = BiggestInt 1 # typeOfIter
@@ -49,26 +49,50 @@ proc semTypeOf(c: PContext; n: PNode): PNode =
     else:
       m = mode.intVal
   result = newNodeI(nkTypeOfExpr, n.info)
+  inc c.inTypeofContext
+  defer: dec c.inTypeofContext # compiles can raise an exception
   let typExpr = semExprWithType(c, n[1], if m == 1: {efInTypeof} else: {})
   result.add typExpr
-  result.typ = makeTypeDesc(c, typExpr.typ)
+  if typExpr.typ.kind == tyFromExpr:
+    typExpr.typ.flags.incl tfNonConstExpr
+  result.typ() = makeTypeDesc(c, typExpr.typ)
 
 type
   SemAsgnMode = enum asgnNormal, noOverloadedSubscript, noOverloadedAsgn
 
 proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode
-proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode
+proc semSubscript(c: PContext, n: PNode, flags: TExprFlags, afterOverloading = false): PNode
 
 proc semArrGet(c: PContext; n: PNode; flags: TExprFlags): PNode =
   result = newNodeI(nkBracketExpr, n.info)
   for i in 1..<n.len: result.add(n[i])
-  result = semSubscript(c, result, flags)
+  result = semSubscript(c, result, flags, afterOverloading = true)
   if result.isNil:
     let x = copyTree(n)
     x[0] = newIdentNode(getIdent(c.cache, "[]"), n.info)
-    bracketNotFoundError(c, x)
-    #localError(c.config, n.info, "could not resolve: " & $n)
-    result = errorNode(c, n)
+    if c.inGenericContext > 0:
+      for i in 0..<n.len:
+        let a = n[i]
+        if a.typ != nil and a.typ.kind in {tyGenericParam, tyFromExpr}:
+          # expression is compiled early in a generic body
+          result = semGenericStmt(c, x)
+          result.typ() = makeTypeFromExpr(c, copyTree(result))
+          result.typ.flags.incl tfNonConstExpr
+          return
+    let s = # extract sym from first arg
+      if n.len > 1:
+        if n[1].kind == nkSym: n[1].sym
+        elif n[1].kind in nkSymChoices + {nkOpenSym} and n[1].len != 0:
+          n[1][0].sym
+        else: nil
+      else: nil
+    if s != nil and s.kind in routineKinds:
+      # this is a failed generic instantiation
+      # semSubscript should already error but this is better for cascading errors
+      result = explicitGenericInstError(c, n)
+    else:
+      bracketNotFoundError(c, x, flags)
+      result = errorNode(c, n)
 
 proc semArrPut(c: PContext; n: PNode; flags: TExprFlags): PNode =
   # rewrite `[]=`(a, i, x)  back to ``a[i] = x``.
@@ -176,15 +200,15 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym)
       let preferStr = traitCall[2].strVal
       prefer = parseEnum[TPreferedDesc](preferStr)
     result = newStrNode(nkStrLit, operand.typeToString(prefer))
-    result.typ = getSysType(c.graph, traitCall[1].info, tyString)
+    result.typ() = getSysType(c.graph, traitCall[1].info, tyString)
     result.info = traitCall.info
   of "name", "$":
     result = newStrNode(nkStrLit, operand.typeToString(preferTypeName))
-    result.typ = getSysType(c.graph, traitCall[1].info, tyString)
+    result.typ() = getSysType(c.graph, traitCall[1].info, tyString)
     result.info = traitCall.info
   of "arity":
     result = newIntNode(nkIntLit, operand.len - ord(operand.kind==tyProc))
-    result.typ = newType(tyInt, c.idgen, context)
+    result.typ() = newType(tyInt, c.idgen, context)
     result.info = traitCall.info
   of "genericHead":
     var arg = operand
@@ -224,8 +248,9 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym)
   of "rangeBase":
     # return the base type of a range type
     var arg = operand.skipTypes({tyGenericInst})
-    assert arg.kind == tyRange
-    result = getTypeDescNode(c, arg.base, operand.owner, traitCall.info)
+    if arg.kind == tyRange:
+      arg = arg.base
+    result = getTypeDescNode(c, arg, operand.owner, traitCall.info)
   of "isCyclic":
     var operand = operand.skipTypes({tyGenericInst})
     let isCyclic = canFormAcycle(c.graph, operand)
@@ -237,7 +262,7 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym)
 proc semTypeTraits(c: PContext, n: PNode): PNode =
   checkMinSonsLen(n, 2, c.config)
   let t = n[1].typ
-  internalAssert c.config, t != nil and t.kind == tyTypeDesc
+  internalAssert c.config, t != nil and t.skipTypes({tyAlias}).kind == tyTypeDesc
   if t.len > 0:
     # This is either a type known to sem or a typedesc
     # param to a regular proc (again, known at instantiation)
@@ -253,7 +278,7 @@ proc semOrd(c: PContext, n: PNode): PNode =
     discard
   else:
     localError(c.config, n.info, errOrdinalTypeExpected % typeToString(parType, preferDesc))
-    result.typ = errorType(c)
+    result.typ() = errorType(c)
 
 proc semBindSym(c: PContext, n: PNode): PNode =
   result = copyNode(n)
@@ -369,7 +394,7 @@ proc semOf(c: PContext, n: PNode): PNode =
         message(c.config, n.info, hintConditionAlwaysTrue, renderTree(n))
         result = newIntNode(nkIntLit, 1)
         result.info = n.info
-        result.typ = getSysType(c.graph, n.info, tyBool)
+        result.typ() = getSysType(c.graph, n.info, tyBool)
         return result
       elif diff == high(int):
         if commonSuperclass(a, b) == nil:
@@ -378,10 +403,10 @@ proc semOf(c: PContext, n: PNode): PNode =
           message(c.config, n.info, hintConditionAlwaysFalse, renderTree(n))
           result = newIntNode(nkIntLit, 0)
           result.info = n.info
-          result.typ = getSysType(c.graph, n.info, tyBool)
+          result.typ() = getSysType(c.graph, n.info, tyBool)
   else:
     localError(c.config, n.info, "'of' takes 2 arguments")
-  n.typ = getSysType(c.graph, n.info, tyBool)
+  n.typ() = getSysType(c.graph, n.info, tyBool)
   result = n
 
 proc semUnown(c: PContext; n: PNode): PNode =
@@ -416,9 +441,9 @@ proc semUnown(c: PContext; n: PNode): PNode =
       result = t
 
   result = copyTree(n[1])
-  result.typ = unownedType(c, result.typ)
+  result.typ() = unownedType(c, result.typ)
   # little hack for injectdestructors.nim (see bug #11350):
-  #result[0].typ = nil
+  #result[0].typ() = nil
 
 proc turnFinalizerIntoDestructor(c: PContext; orig: PSym; info: TLineInfo): PSym =
   # We need to do 2 things: Replace n.typ which is a 'ref T' by a 'var T' type.
@@ -428,7 +453,7 @@ proc turnFinalizerIntoDestructor(c: PContext; orig: PSym; info: TLineInfo): PSym
   proc transform(c: PContext; n: PNode; old, fresh: PType; oldParam, newParam: PSym): PNode =
     result = shallowCopy(n)
     if sameTypeOrNil(n.typ, old):
-      result.typ = fresh
+      result.typ() = fresh
     if n.kind == nkSym and n.sym == oldParam:
       result.sym = newParam
     for i in 0 ..< safeLen(n):
@@ -439,7 +464,7 @@ proc turnFinalizerIntoDestructor(c: PContext; orig: PSym; info: TLineInfo): PSym
   result = copySym(orig, c.idgen)
   result.info = info
   result.flags.incl sfFromGeneric
-  result.owner = orig
+  setOwner(result, orig)
   let origParamType = orig.typ.firstParamType
   let newParamType = makeVarType(result, origParamType.skipTypes(abstractPtrs), c.idgen)
   let oldParam = orig.typ.n[1].sym
@@ -510,32 +535,34 @@ proc semNewFinalize(c: PContext; n: PNode): PNode =
         discard "already turned this one into a finalizer"
       else:
         if fin.instantiatedFrom != nil and fin.instantiatedFrom != fin.owner: #undo move
-          fin.owner = fin.instantiatedFrom
-        let wrapperSym = newSym(skProc, getIdent(c.graph.cache, fin.name.s & "FinalizerWrapper"), c.idgen, fin.owner, fin.info)
-        let selfSymNode = newSymNode(copySym(fin.ast[paramsPos][1][0].sym, c.idgen))
-        selfSymNode.typ = fin.typ.firstParamType
-        wrapperSym.flags.incl sfUsed
+          setOwner(fin, fin.instantiatedFrom)
 
-        let wrapper = c.semExpr(c, newProcNode(nkProcDef, fin.info, body = newTree(nkCall, newSymNode(fin), selfSymNode),
-          params = nkFormalParams.newTree(c.graph.emptyNode,
-                  newTree(nkIdentDefs, selfSymNode, newNodeIT(nkType,
-                  fin.ast[paramsPos][1][1].info, fin.typ.firstParamType), c.graph.emptyNode)
-                  ),
-          name = newSymNode(wrapperSym), pattern = fin.ast[patternPos],
-          genericParams = fin.ast[genericParamsPos], pragmas = fin.ast[pragmasPos], exceptions = fin.ast[miscPos]), {})
+        if fin.typ[1].skipTypes(abstractInst).kind != tyRef:
+          bindTypeHook(c, fin, n, attachedDestructor)
+        else:
+          let wrapperSym = newSym(skProc, getIdent(c.graph.cache, fin.name.s & "FinalizerWrapper"), c.idgen, fin.owner, fin.info)
+          let selfSymNode = newSymNode(copySym(fin.ast[paramsPos][1][0].sym, c.idgen))
+          selfSymNode.typ() = fin.typ.firstParamType
+          wrapperSym.flags.incl sfUsed
 
-        var transFormedSym = turnFinalizerIntoDestructor(c, wrapperSym, wrapper.info)
-        transFormedSym.owner = fin
-        if c.config.backend == backendCpp or sfCompileToCpp in c.module.flags:
-          let origParamType = transFormedSym.ast[bodyPos][1].typ
-          let selfSymbolType = makePtrType(c, origParamType.skipTypes(abstractPtrs))
-          let selfPtr = newNodeI(nkHiddenAddr, transFormedSym.ast[bodyPos][1].info)
-          selfPtr.add transFormedSym.ast[bodyPos][1]
-          selfPtr.typ = selfSymbolType
-          transFormedSym.ast[bodyPos][1] = c.semExpr(c, selfPtr)
-        # TODO: suppress var destructor warnings; if newFinalizer is not
-        # TODO: deprecated, try to implement plain T destructor
-        bindTypeHook(c, transFormedSym, n, attachedDestructor, suppressVarDestructorWarning = true)
+          let wrapper = c.semExpr(c, newProcNode(nkProcDef, fin.info, body = newTree(nkCall, newSymNode(fin), selfSymNode),
+            params = nkFormalParams.newTree(c.graph.emptyNode,
+                    newTree(nkIdentDefs, selfSymNode, newNodeIT(nkType,
+                    fin.ast[paramsPos][1][1].info, fin.typ.firstParamType), c.graph.emptyNode)
+                    ),
+            name = newSymNode(wrapperSym), pattern = fin.ast[patternPos],
+            genericParams = fin.ast[genericParamsPos], pragmas = fin.ast[pragmasPos], exceptions = fin.ast[miscPos]), {})
+
+          var transFormedSym = turnFinalizerIntoDestructor(c, wrapperSym, wrapper.info)
+          setOwner(transFormedSym, fin)
+          if c.config.backend == backendCpp or sfCompileToCpp in c.module.flags:
+            let origParamType = transFormedSym.ast[bodyPos][1].typ
+            let selfSymbolType = makePtrType(c, origParamType.skipTypes(abstractPtrs))
+            let selfPtr = newNodeI(nkHiddenAddr, transFormedSym.ast[bodyPos][1].info)
+            selfPtr.add transFormedSym.ast[bodyPos][1]
+            selfPtr.typ() = selfSymbolType
+            transFormedSym.ast[bodyPos][1] = c.semExpr(c, selfPtr)
+          bindTypeHook(c, transFormedSym, n, attachedDestructor)
   result = addDefaultFieldForNew(c, n)
 
 proc semPrivateAccess(c: PContext, n: PNode): PNode =
@@ -587,7 +614,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   of mTypeTrait: result = semTypeTraits(c, n)
   of mAstToStr:
     result = newStrNodeT(renderTree(n[1], {renderNoComments}), n, c.graph)
-    result.typ = getSysType(c.graph, n.info, tyString)
+    result.typ() = getSysType(c.graph, n.info, tyString)
   of mInstantiationInfo: result = semInstantiationInfo(c, n)
   of mOrd: result = semOrd(c, n)
   of mOf: result = semOf(c, n)
@@ -600,7 +627,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
       result = semDynamicBindSym(c, n)
   of mProcCall:
     result = n
-    result.typ = n[1].typ
+    result.typ() = n[1].typ
   of mDotDot:
     result = n
   of mPlugin:
@@ -643,7 +670,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
       result[0] = newSymNode(op)
       if op.typ.len == 3:
         let boolLit = newIntLit(c.graph, n.info, 1)
-        boolLit.typ = getSysType(c.graph, n.info, tyBool)
+        boolLit.typ() = getSysType(c.graph, n.info, tyBool)
         result.add boolLit
   of mWasMoved:
     result = n
@@ -685,7 +712,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     result = n
     if result.typ != nil and expectedType != nil and result.typ.kind == tySequence and
         expectedType.kind == tySequence and result.typ.elementType.kind == tyEmpty:
-      result.typ = expectedType # type inference for empty sequence # bug #21377
+      result.typ() = expectedType # type inference for empty sequence # bug #21377
   of mEnsureMove:
     result = n
     if n[1].kind in {nkStmtListExpr, nkBlockExpr,