allows foreign functions having forward decls

#23823

.github/workflows/ci_bench.yml

@@ -1,115 +0,0 @@
-name: Benchmarks CI
-on:
-  pull_request:
-  push:
-    branches:
-      - 'devel'
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-20.04]
-        cpu: [amd64]
-    name: '${{ matrix.os }}'
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 60 # refs bug #18178
-    steps:
-      - name: 'Checkout'
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-
-      - name: 'Install node.js 20.x'
-        uses: actions/setup-node@v4
-        with:
-          node-version: '20.x'
-
-      - name: 'Install dependencies (Linux amd64)'
-        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
-        run: |
-          sudo apt-fast update -qq
-          DEBIAN_FRONTEND='noninteractive' \
-            sudo apt-fast install --no-install-recommends -yq \
-              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
-              valgrind libc6-dbg libblas-dev xorg-dev
-
-      - name: 'Add build binaries to PATH'
-        shell: bash
-        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
-
-      - name: 'Build csourcesAny'
-        shell: bash
-        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
-
-      - name: 'Build koch'
-        shell: bash
-        run: nim c koch
-
-      - name: 'Build Nim'
-        shell: bash
-        run: ./koch boot -d:release -d:nimStrictMode --lib:lib
-
-      - name: 'Build Nimble'
-        shell: bash
-        run: ./koch nimble
-
-      - name: 'Action'
-        shell: bash
-        run: nim c -r -d:release ci/action.nim
-
-      - name: 'Checkout minimize'
-        uses: actions/checkout@v4
-        with:
-          repository: 'nim-lang/ci_bench'
-          path: minimize
-
-      - name: 'Run minimize benchmarks'
-        shell: bash
-        run: ./minimize/minimize ci-bench
-
-      - name: 'Restore minimize cached database'
-        uses: actions/cache/restore@v3
-        with:
-          path: minimize.csv
-          key: minimize-db-key
-
-      - name: 'Update minimize db'
-        shell: bash
-        run: ./minimize/minimize update-db
-
-      - name: 'Save minimize cached database'
-        if: |
-          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
-          matrix.target == 'linux'
-        uses: actions/cache/save@v3
-        with:
-          path: minimize.csv
-          key: minimize-db-key
-
-      - name: 'Generate minimize report'
-        shell: bash
-        run: ./minimize/minimize generate-report
-
-      - name: 'Archive minimize report'
-        uses: actions/upload-artifact@v3
-        with:
-          name: minimize-report
-          path: |
-            minimize/minimize.html
-            minimize/minimize.csv
-
-      # Requires additional permissions, see:
-      # https://github.com/nim-lang/Nim/actions/runs/4778177321/jobs/8494423792?pr=21566
-      # - name: 'Publish HTML report'
-      #   uses: rossjrw/pr-preview-action@v1
-      #   with:
-      #     source-dir: minimize
-      #     umbrella-dir: minimize
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract md summary
-        run: |
-          cat minimize/summary.md  >> $GITHUB_STEP_SUMMARY

azure-pipelines.yml

@@ -22,10 +22,6 @@ jobs:
       Linux_amd64:
         vmImage: 'ubuntu-20.04'
         CPU: amd64
-
-      Linux_amd64_gcc14:
-        vmImage: 'ubuntu-24.04'
-        CPU: amd64
       # regularly breaks, refs bug #17325
 #       Linux_i386:
 #         # on 'ubuntu-16.04' (not supported anymore anyways) it errored with:
@@ -91,15 +87,6 @@ jobs:
       displayName: 'Install dependencies (amd64 Linux)'
       condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
 
-    - bash: |
-        sudo add-apt-repository ppa:ubuntu-toolchain-r/ppa -y
-        sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
-        sudo apt update
-        sudo apt install gcc-14 g++-14
-        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 60 --slave /usr/bin/g++ g++ /usr/bin/g++-14
-      displayName: 'Install dependencies (amd64 Linux gcc 14)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'), eq(variables['vmImage'], 'ubuntu-latest'))
-
     - bash: |
         set -e
         . ci/funs.sh

changelog.md

@@ -17,6 +17,10 @@
 
 - `bindMethod` in `std/jsffi` is deprecated, don't use it with closures.
 
+- JS backend now supports lambda lifting for closures. Use `--legacy:jsNoLambdaLifting` to emulate old behavior.
+
+- `owner` in `std/macros` is deprecated.
+
 ## Standard library additions and changes
 
 [//]: # "Changes:"
@@ -28,7 +32,7 @@
 [//]: # "Additions:"
 
 - Added `newStringUninit` to system, which creates a new string of length `len` like `newString` but with uninitialized content.
-- Added `setLenUninit` to system, which doesn't initalize
+- Added `setLenUninit` to system, which doesn't initialize
 slots when enlarging a sequence.
 - Added `hasDefaultValue` to `std/typetraits` to check if a type has a valid default value.
 - Added Viewport API for the JavaScript targets in the `dom` module.
@@ -37,6 +41,12 @@ slots when enlarging a sequence.
   objects the cyclic collector did free. If the number is zero that is a strong indicator that you can use `--mm:arc`
   instead of `--mm:orc`.
 - A `$` template is provided for `Path` in `std/paths`.
+- `std/hashes.hash(x:string)` changed to produce a 64-bit string `Hash` (based
+on Google's Farm Hash) which is also often faster than the present one.  Define
+`nimStringHash2` to get the old values back.  `--jsbigint=off` mode always only
+produces the old values.  This may impact your automated tests if they depend
+on hash order in some obvious or indirect way.  Using `sorted` or `OrderedTable`
+is often an easy workaround.
 
 [//]: # "Deprecations:"
 

compiler/ast.nim

@@ -41,7 +41,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # 51 flags!
+  TSymFlag* = enum    # 53 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -52,6 +52,7 @@ type
     sfForward,        # symbol is forward declared
     sfWasForwarded,   # symbol had a forward declaration
                       # (implies it's too dangerous to patch its type signature)
+    sfNosinks,        # symbol has a `nosinks` pragma
     sfImportc,        # symbol is external; imported
     sfExportc,        # symbol is exported (under a specified name)
     sfMangleCpp,      # mangle as cpp (combines with `sfExportc`)
@@ -126,6 +127,8 @@ type
     sfByCopy          # param is marked as pass bycopy
     sfMember          # proc is a C++ member of a type
     sfCodegenDecl     # type, proc, global or proc param is marked as codegenDecl
+    sfWasGenSym       # symbol was 'gensym'ed
+    sfForceLift       # variable has to be lifted into closure environment
 
   TSymFlags* = set[TSymFlag]
 

compiler/ccgcalls.nim

@@ -76,6 +76,23 @@ proc isHarmlessStore(p: BProc; canRaise: bool; d: TLoc): bool =
   else:
     result = false
 
+proc cleanupTemp(p: BProc; returnType: PType, tmp: TLoc): bool =
+  if returnType.kind in {tyVar, tyLent}:
+    # we don't need to worry about var/lent return types
+    result = false
+  elif hasDestructor(returnType) and getAttachedOp(p.module.g.graph, returnType, attachedDestructor) != nil:
+    let dtor = getAttachedOp(p.module.g.graph, returnType, attachedDestructor)
+    var op = initLocExpr(p, newSymNode(dtor))
+    var callee = rdLoc(op)
+    let destroy = if dtor.typ.firstParamType.kind == tyVar:
+        callee & "(&" & rdLoc(tmp) & ")"
+      else:
+        callee & "(" & rdLoc(tmp) & ")"
+    raiseExitCleanup(p, destroy)
+    result = true
+  else:
+    result = false
+
 proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
@@ -128,25 +145,32 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
             if canRaise: raiseExit(p)
 
       elif isHarmlessStore(p, canRaise, d):
-        if d.k == locNone: d = getTemp(p, typ.returnType)
+        var useTemp = false
+        if d.k == locNone:
+          useTemp = true
+          d = getTemp(p, typ.returnType)
         assert(d.t != nil)        # generate an assignment to d:
         var list = initLoc(locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, flags) # no need for deep copying
-        if canRaise: raiseExit(p)
+        if canRaise:
+          if not (useTemp and cleanupTemp(p, typ.returnType, d)):
+            raiseExit(p)
       else:
         var tmp: TLoc = getTemp(p, typ.returnType, needsInit=true)
         var list = initLoc(locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, tmp, list, flags) # no need for deep copying
-        if canRaise: raiseExit(p)
+        if canRaise:
+          if not cleanupTemp(p, typ.returnType, tmp):
+            raiseExit(p)
         genAssignment(p, d, tmp, {})
   else:
     pl.add(");\n")
     line(p, cpsStmts, pl)
     if canRaise: raiseExit(p)
 
-proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
+proc genBoundsCheck(p: BProc; arr, a, b: TLoc; arrTyp: PType)
 
 proc reifiedOpenArray(n: PNode): bool {.inline.} =
   var x = n
@@ -167,15 +191,15 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF
   var a = initLocExpr(p, q[1])
   var b = initLocExpr(p, q[2])
   var c = initLocExpr(p, q[3])
-  # but first produce the required index checks:
-  if optBoundsCheck in p.options:
-    genBoundsCheck(p, a, b, c)
-  if prepareForMutation:
-    linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
   # bug #23321: In the function mapType, ptrs (tyPtr, tyVar, tyLent, tyRef)
   # are mapped into ctPtrToArray, the dereference of which is skipped
-  # in the `genref`. We need to skip these ptrs here
+  # in the `genDeref`. We need to skip these ptrs here
   let ty = skipTypes(a.t, abstractVar+{tyPtr, tyRef})
+  # but first produce the required index checks:
+  if optBoundsCheck in p.options:
+    genBoundsCheck(p, a, b, c, ty)
+  if prepareForMutation:
+    linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
   let dest = getTypeDesc(p.module, destType)
   let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)]
   case ty.kind
@@ -307,18 +331,33 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Rope; need
       addAddrLoc(p.config, withTmpIfNeeded(p, a, needsTmp), result)
   elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
-    a = initLocExprSingleUse(p, n[0])
+    # bug #23748: we need to introduce a temporary here. The expression type
+    # will be a reference in C++ and we cannot create a temporary reference
+    # variable. Thus, we create a temporary pointer variable instead.
+    let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray
+    if needsIndirect:
+      n.typ = n.typ.exactReplica
+      n.typ.flags.incl tfVarIsPtr
+    a = initLocExprSingleUse(p, n)
+    a = withTmpIfNeeded(p, a, needsTmp)
+    if needsIndirect: a.flags.incl lfIndirect
     # if the proc is 'importc'ed but not 'importcpp'ed then 'var T' still
     # means '*T'. See posix.nim for lots of examples that do that in the wild.
     let callee = call[0]
     if callee.kind == nkSym and
         {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
-        {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
+        {lfHeader, lfNoDecl} * callee.sym.loc.flags != {} and
+        needsIndirect:
       addAddrLoc(p.config, a, result)
     else:
       addRdLoc(a, result)
   else:
     a = initLocExprSingleUse(p, n)
+    if param.typ.kind in abstractPtrs:
+      let typ = skipTypes(param.typ, abstractPtrs)
+      if typ.sym != nil and sfImportc in typ.sym.flags:
+        a.r = "(($1) ($2))" %
+          [getTypeDesc(p.module, param.typ), rdCharLoc(a)]
     addRdLoc(withTmpIfNeeded(p, a, needsTmp), result)
   #assert result != nil
 
@@ -401,9 +440,11 @@ proc genParams(p: BProc, ri: PNode, typ: PType; result: var Rope) =
         if not needTmp[i - 1]:
           needTmp[i - 1] = potentialAlias(n, potentialWrites)
       getPotentialWrites(ri[i], false, potentialWrites)
-    if ri[i].kind in {nkHiddenAddr, nkAddr}:
-      # Optimization: don't use a temp, if we would only take the address anyway
-      needTmp[i - 1] = false
+    when false:
+      # this optimization is wrong, see bug #23748
+      if ri[i].kind in {nkHiddenAddr, nkAddr}:
+        # Optimization: don't use a temp, if we would only take the address anyway
+        needTmp[i - 1] = false
 
   var oldLen = result.len
   for i in 1..<ri.len:

compiler/ccgexprs.nim

@@ -343,7 +343,7 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
   of tyString:
     if optSeqDestructors in p.config.globalOptions:
       genGenericAsgn(p, dest, src, flags)
-    elif (needToCopy notin flags and src.storage != OnStatic) or canMove(p, src.lode, dest):
+    elif ({needToCopy, needToCopySinkParam} * flags == {} and src.storage != OnStatic) or canMove(p, src.lode, dest):
       genRefAssign(p, dest, src)
     else:
       if (dest.storage == OnStack and p.config.selectedGC != gcGo) or not usesWriteBarrier(p.config):
@@ -812,6 +812,8 @@ proc genAddr(p: BProc, e: PNode, d: var TLoc) =
     #Message(e.info, warnUser, "HERE NEW &")
   elif mapType(p.config, e[0].typ, mapTypeChooser(e[0]) == skParam) == ctArray or isCppRef(p, e.typ):
     expr(p, e[0], d)
+    # bug #19497
+    d.lode = e
   else:
     var a: TLoc = initLocExpr(p, e[0])
     putIntoDest(p, d, e, addrLoc(p.config, a), a.storage)
@@ -1019,8 +1021,8 @@ proc genCStringElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   putIntoDest(p, d, n,
               ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage)
 
-proc genBoundsCheck(p: BProc; arr, a, b: TLoc) =
-  let ty = skipTypes(arr.t, abstractVarRange)
+proc genBoundsCheck(p: BProc; arr, a, b: TLoc; arrTyp: PType) =
+  let ty = arrTyp
   case ty.kind
   of tyOpenArray, tyVarargs:
     if reifiedOpenArray(arr.lode):
@@ -1855,7 +1857,7 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       var b = initLocExpr(p, a[2])
       var c = initLocExpr(p, a[3])
       if optBoundsCheck in p.options:
-        genBoundsCheck(p, m, b, c)
+        genBoundsCheck(p, m, b, c, skipTypes(m.t, abstractVarRange))
       if op == mHigh:
         putIntoDest(p, d, e, ropecg(p.module, "(($2)-($1))", [rdLoc(b), rdLoc(c)]))
       else:
@@ -2345,8 +2347,13 @@ proc genMove(p: BProc; n: PNode; d: var TLoc) =
           else:
             linefmt(p, cpsStmts, "$1($2);$n", [rdLoc(b), byRefLoc(p, a)])
     else:
-      let flags = if not canMove(p, n[1], d): {needToCopy} else: {}
-      genAssignment(p, d, a, flags)
+      if n[1].kind == nkSym and isSinkParam(n[1].sym):
+        var tmp = getTemp(p, n[1].typ.skipTypes({tySink}))
+        genAssignment(p, tmp, a, {needToCopySinkParam})
+        genAssignment(p, d, tmp, {})
+        resetLoc(p, tmp)
+      else:
+        genAssignment(p, d, a, {})
       resetLoc(p, a)
 
 proc genDestroy(p: BProc; n: PNode) =
@@ -3239,7 +3246,8 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
       getNullValueAux(p, t, it, constOrNil, result, count, isConst, info)
   of nkRecCase:
     getNullValueAux(p, t, obj[0], constOrNil, result, count, isConst, info)
-    if count > 0: result.add ", "
+    var res = ""
+    if count > 0: res.add ", "
     var branch = Zero
     if constOrNil != nil:
       ## find kind value, default is zero if not specified
@@ -3253,18 +3261,21 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
           break
 
     let selectedBranch = caseObjDefaultBranch(obj, branch)
-    result.add "{"
+    res.add "{"
     var countB = 0
     let b = lastSon(obj[selectedBranch])
     # designated initilization is the only way to init non first element of unions
     # branches are allowed to have no members (b.len == 0), in this case they don't need initializer
     if b.kind == nkRecList and not isEmptyCaseObjectBranch(b):
-      result.add "._" & mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
-      getNullValueAux(p, t,  b, constOrNil, result, countB, isConst, info)
-      result.add "}"
+      res.add "._" & mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
+      getNullValueAux(p, t,  b, constOrNil, res, countB, isConst, info)
+      res.add "}"
     elif b.kind == nkSym:
-      result.add "." & mangleRecFieldName(p.module, b.sym) & " = "
-      getNullValueAux(p, t,  b, constOrNil, result, countB, isConst, info)
+      res.add "." & mangleRecFieldName(p.module, b.sym) & " = "
+      getNullValueAux(p, t,  b, constOrNil, res, countB, isConst, info)
+    else:
+      return
+    result.add res
     result.add "}"
 
   of nkSym:

compiler/ccgstmts.nim

@@ -289,7 +289,7 @@ proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Rope) =
     #echo "New code produced for ", v.name.s, " ", p.config $ value.info
     genBracedInit(p, value, isConst = false, v.typ, result)
 
-proc genCppParamsForCtor(p: BProc; call: PNode): string =
+proc genCppParamsForCtor(p: BProc; call: PNode; didGenTemp: var bool): string =
   result = ""
   var argsCounter = 0
   let typ = skipTypes(call[0].typ, abstractInst)
@@ -298,12 +298,23 @@ proc genCppParamsForCtor(p: BProc; call: PNode): string =
     #if it's a type we can just generate here another initializer as we are in an initializer context
     if call[i].kind == nkCall and call[i][0].kind == nkSym and call[i][0].sym.kind == skType:
       if argsCounter > 0: result.add ","
-      result.add genCppInitializer(p.module, p, call[i][0].sym.typ)
+      result.add genCppInitializer(p.module, p, call[i][0].sym.typ, didGenTemp)
     else:
+      #We need to test for temp in globals, see: #23657
+      let param =
+        if typ[i].kind in {tyVar} and call[i].kind == nkHiddenAddr:
+          call[i][0]
+        else:
+          call[i]
+      if param.kind != nkBracketExpr or param.typ.kind in
+        {tyRef, tyPtr, tyUncheckedArray, tyArray, tyOpenArray,
+          tyVarargs, tySequence, tyString, tyCstring, tyTuple}:
+        let tempLoc = initLocExprSingleUse(p, param)
+        didGenTemp = didGenTemp or tempLoc.k == locTemp
       genOtherArg(p, call, i, typ, result, argsCounter)
 
-proc genCppVarForCtor(p: BProc; call: PNode; decl: var Rope) =
-  let params = genCppParamsForCtor(p, call)
+proc genCppVarForCtor(p: BProc; call: PNode; decl: var Rope, didGenTemp: var bool) =
+  let params = genCppParamsForCtor(p, call, didGenTemp)
   if params.len == 0:
     decl = runtimeFormat("$#;\n", [decl])
   else:
@@ -330,7 +341,14 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
       # v.owner.kind != skModule:
       targetProc = p.module.preInitProc
     if isCppCtorCall and not containsHiddenPointer(v.typ):
-      callGlobalVarCppCtor(targetProc, v, vn, value)
+      var didGenTemp = false
+      callGlobalVarCppCtor(targetProc, v, vn, value, didGenTemp)
+      if didGenTemp:
+        message(p.config, vn.info, warnGlobalVarConstructorTemporary, vn.sym.name.s)
+        #We fail to call the constructor in the global scope so we do the call inside the main proc
+        assignGlobalVar(targetProc, vn, valueAsRope)
+        var loc = initLocExprSingleUse(targetProc, value)
+        genAssignment(targetProc, v.loc, loc, {})
     else:
       assignGlobalVar(targetProc, vn, valueAsRope)
 
@@ -365,7 +383,8 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
       var decl = localVarDecl(p, vn)
       var tmp: TLoc
       if isCppCtorCall:
-        genCppVarForCtor(p, value, decl)
+        var didGenTemp = false
+        genCppVarForCtor(p, value, decl, didGenTemp)
         line(p, cpsStmts, decl)
       else:
         tmp = initLocExprSingleUse(p, value)
@@ -736,6 +755,18 @@ proc raiseExit(p: BProc) =
       lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
         [p.nestedTryStmts[^1].label])
 
+proc raiseExitCleanup(p: BProc, destroy: string) =
+  assert p.config.exc == excGoto
+  if nimErrorFlagDisabled notin p.flags:
+    p.flags.incl nimErrorFlagAccessed
+    if p.nestedTryStmts.len == 0:
+      p.flags.incl beforeRetNeeded
+      # easy case, simply goto 'ret':
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) {$1; goto BeforeRet_;}$n", [destroy])
+    else:
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) {$2; goto LA$1_;}$n",
+        [p.nestedTryStmts[^1].label, destroy])
+
 proc finallyActions(p: BProc) =
   if p.config.exc != excGoto and p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept:
     # if the current try stmt have a finally block,
@@ -765,10 +796,14 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     var e = rdLoc(a)
     discard getTypeDesc(p.module, t[0].typ)
     var typ = skipTypes(t[0].typ, abstractPtrs)
-    # XXX For reasons that currently escape me, this is only required by the new
-    # C++ based exception handling:
-    if p.config.exc == excCpp:
+    case p.config.exc
+    of excCpp:
       blockLeaveActions(p, howManyTrys = 0, howManyExcepts = p.inExceptBlockLen)
+    of excGoto:
+      blockLeaveActions(p, howManyTrys = 0,
+        howManyExcepts = (if p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept: 1 else: 0))
+    else:
+      discard
     genLineDir(p, t)
     if isImportedException(typ, p.config):
       lineF(p, cpsStmts, "throw $1;$n", [e])
@@ -1547,14 +1582,14 @@ proc genAsmStmt(p: BProc, t: PNode) =
       if whichPragma(i) == wAsmSyntax:
         asmSyntax = i[1].strVal
 
-  if asmSyntax != "" and 
+  if asmSyntax != "" and
      not (
       asmSyntax == "gcc" and hasGnuAsm in CC[p.config.cCompiler].props or
       asmSyntax == "vcc" and hasGnuAsm notin CC[p.config.cCompiler].props):
     localError(
-      p.config, t.info, 
+      p.config, t.info,
       "Your compiler does not support the specified inline assembler")
-  
+
   genAsmOrEmitStmt(p, t, isAsmStmt=true, s)
   # see bug #2362, "top level asm statements" seem to be a mis-feature
   # but even if we don't do this, the example in #2362 cannot possibly

compiler/ccgtypes.nim

@@ -277,9 +277,12 @@ proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool =
       if rettype.isImportedCppType or t.isImportedCppType or
           (typ.callConv == ccCDecl and conf.selectedGC in {gcArc, gcAtomicArc, gcOrc}):
         # prevents nrvo for cdecl procs; # bug #23401
-        return false
-      result = containsGarbageCollectedRef(t) or
-          (t.kind == tyObject and not isObjLackingTypeField(t))
+        result = false
+      else:
+        result = containsGarbageCollectedRef(t) or
+            (t.kind == tyObject and not isObjLackingTypeField(t)) or
+            (getSize(conf, rettype) == szUnknownSize and (t.sym == nil or sfImportc notin t.sym.flags))
+
     else: result = false
 
 const
@@ -678,9 +681,9 @@ proc hasCppCtor(m: BModule; typ: PType): bool =
       if sfConstructor in prc.flags:
         return true
 
-proc genCppParamsForCtor(p: BProc; call: PNode): string
+proc genCppParamsForCtor(p: BProc; call: PNode; didGenTemp: var bool): string
 
-proc genCppInitializer(m: BModule, prc: BProc; typ: PType): string =
+proc genCppInitializer(m: BModule, prc: BProc; typ: PType; didGenTemp: var bool): string =
   #To avoid creating a BProc per test when called inside a struct nil BProc is allowed
   result = "{}"
   if typ.itemId in m.g.graph.initializersPerType:
@@ -689,7 +692,7 @@ proc genCppInitializer(m: BModule, prc: BProc; typ: PType): string =
       var p = prc
       if p == nil:
         p = BProc(module: m)
-      result = "{" & genCppParamsForCtor(p, call) & "}"
+      result = "{" & genCppParamsForCtor(p, call, didGenTemp) & "}"
       if prc == nil:
         assert p.blocks.len == 0, "BProc belongs to a struct doesnt have blocks"
 
@@ -759,7 +762,8 @@ proc genRecordFieldsAux(m: BModule; n: PNode,
         # tyGenericInst for C++ template support
         let noInit = sfNoInit in field.flags or (field.typ.sym != nil and sfNoInit in field.typ.sym.flags)
         if not noInit and (fieldType.isOrHasImportedCppType() or hasCppCtor(m, field.owner.typ)):
-          var initializer = genCppInitializer(m, nil, fieldType)
+          var didGenTemp = false
+          var initializer = genCppInitializer(m, nil, fieldType, didGenTemp)
           result.addf("\t$1$3 $2$4;$n", [getTypeDescAux(m, field.loc.t, check, dkField), sname, noAlias, initializer])
         else:
           result.addf("\t$1$3 $2;$n", [getTypeDescAux(m, field.loc.t, check, dkField), sname, noAlias])
@@ -1471,7 +1475,7 @@ proc genObjectInfo(m: BModule; typ, origType: PType, name: Rope; info: TLineInfo
                       typeToString(typ))
   genTypeInfoAux(m, typ, origType, name, info)
   var tmp = getNimNode(m)
-  if not isImportedType(typ):
+  if (not isImportedType(typ)) or tfCompleteStruct in typ.flags:
     genObjectFields(m, typ, origType, typ.n, tmp, info)
   m.s[cfsTypeInit3].addf("$1.node = &$2;$n", [tiNameForHcr(m, name), tmp])
   var t = typ.baseClass

compiler/ccgutils.nim

@@ -90,6 +90,9 @@ proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool =
     if s.typ.sym != nil and sfForward in s.typ.sym.flags:
       # forwarded objects are *always* passed by pointers for consistency!
       result = true
+    elif s.typ.kind == tySink and conf.selectedGC notin {gcArc, gcAtomicArc, gcOrc, gcHooks}:
+      # bug #23354:
+      result = false
     elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
       result = true           # requested anyway
     elif (tfFinal in pt.flags) and (pt[0] == nil):

compiler/cgen.nim

@@ -412,6 +412,7 @@ proc rdCharLoc(a: TLoc): Rope =
 type
   TAssignmentFlag = enum
     needToCopy
+    needToCopySinkParam
     needTempForOpenArray
   TAssignmentFlags = set[TAssignmentFlag]
 
@@ -562,8 +563,9 @@ proc getTemp(p: BProc, t: PType, needsInit=false): TLoc =
   result = TLoc(r: "T" & rope(p.labels) & "_", k: locTemp, lode: lodeTyp t,
                 storage: OnStack, flags: {})
   if p.module.compileToCpp and isOrHasImportedCppType(t):
+    var didGenTemp = false
     linefmt(p, cpsLocals, "$1 $2$3;$n", [getTypeDesc(p.module, t, dkVar), result.r,
-      genCppInitializer(p.module, p, t)])
+      genCppInitializer(p.module, p, t, didGenTemp)])
   else:
     linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, dkVar), result.r])
   constructLoc(p, result, not needsInit)
@@ -620,7 +622,8 @@ proc assignLocalVar(p: BProc, n: PNode) =
   let nl = if optLineDir in p.config.options: "" else: "\n"
   var decl = localVarDecl(p, n)
   if p.module.compileToCpp and isOrHasImportedCppType(n.typ):
-    decl.add genCppInitializer(p.module, p, n.typ)
+    var didGenTemp = false
+    decl.add genCppInitializer(p.module, p, n.typ, didGenTemp)
   decl.add ";" & nl
   line(p, cpsLocals, decl)
 
@@ -655,18 +658,7 @@ proc genGlobalVarDecl(p: BProc, n: PNode; td, value: Rope; decl: var Rope) =
     else:
       decl = runtimeFormat(s.cgDeclFrmt & ";$n", [td, s.loc.r])
 
-proc genCppVarForCtor(p: BProc; call: PNode; decl: var Rope)
-
-proc callGlobalVarCppCtor(p: BProc; v: PSym; vn, value: PNode) =
-  let s = vn.sym
-  fillBackendName(p.module, s)
-  fillLoc(s.loc, locGlobalVar, vn, OnHeap)
-  var decl: Rope = ""
-  let td = getTypeDesc(p.module, vn.sym.typ, dkVar)
-  genGlobalVarDecl(p, vn, td, "", decl)
-  decl.add " " & $s.loc.r
-  genCppVarForCtor(p, value, decl)
-  p.module.s[cfsVars].add decl
+proc genCppVarForCtor(p: BProc; call: PNode; decl: var Rope; didGenTemp: var bool)
 
 proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
   let s = n.sym
@@ -722,6 +714,18 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
     # fixes tests/run/tzeroarray:
     resetLoc(p, s.loc)
 
+proc callGlobalVarCppCtor(p: BProc; v: PSym; vn, value: PNode; didGenTemp: var bool) =
+  let s = vn.sym
+  fillBackendName(p.module, s)
+  fillLoc(s.loc, locGlobalVar, vn, OnHeap)
+  var decl: Rope = ""
+  let td = getTypeDesc(p.module, vn.sym.typ, dkVar)
+  genGlobalVarDecl(p, vn, td, "", decl)
+  decl.add " " & $s.loc.r
+  genCppVarForCtor(p, value, decl, didGenTemp)
+  if didGenTemp:  return # generated in the caller
+  p.module.s[cfsVars].add decl
+
 proc assignParam(p: BProc, s: PSym, retType: PType) =
   assert(s.loc.r != "")
   scopeMangledParam(p, s)
@@ -749,6 +753,7 @@ proc intLiteral(i: BiggestInt; result: var Rope)
 proc genLiteral(p: BProc, n: PNode; result: var Rope)
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope; argsCounter: var int)
 proc raiseExit(p: BProc)
+proc raiseExitCleanup(p: BProc, destroy: string)
 
 proc initLocExpr(p: BProc, e: PNode, flags: TLocFlags = {}): TLoc =
   result = initLoc(locNone, e, OnUnknown, flags)
@@ -1070,7 +1075,11 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
       if result != Unknown: return result
   of nkAsgn, nkFastAsgn, nkSinkAsgn:
     if n[0].kind == nkSym and n[0].sym.kind == skResult:
-      if not containsResult(n[1]): result = InitSkippable
+      if not containsResult(n[1]):
+        if allPathsAsgnResult(p, n[1]) == InitRequired:
+          result = InitRequired
+        else:
+          result = InitSkippable
       else: result = InitRequired
     elif containsResult(n):
       result = InitRequired
@@ -1148,6 +1157,10 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
         allPathsInBranch(n[i])
   of nkRaiseStmt:
     result = InitRequired
+  of nkChckRangeF, nkChckRange64, nkChckRange:
+    # TODO: more checks might need to be covered like overflow, indexDefect etc.
+    # bug #22852
+    result = InitRequired
   else:
     for i in 0..<n.safeLen:
       allPathsInBranch(n[i])

compiler/closureiters.nim

@@ -36,13 +36,6 @@
 #  else:
 #    return
 
-# The transformation should play well with lambdalifting, however depending
-# on situation, it can be called either before or after lambdalifting
-# transformation. As such we behave slightly differently, when accessing
-# iterator state, or using temp variables. If lambdalifting did not happen,
-# we just create local variables, so that they will be lifted further on.
-# Otherwise, we utilize existing env, created by lambdalifting.
-
 # Lambdalifting treats :state variable specially, it should always end up
 # as the first field in env. Currently C codegen depends on this behavior.
 
@@ -151,7 +144,6 @@ type
   Ctx = object
     g: ModuleGraph
     fn: PSym
-    stateVarSym: PSym # :state variable. nil if env already introduced by lambdalifting
     tmpResultSym: PSym # Used when we return, but finally has to interfere
     unrollFinallySym: PSym # Indicates that we're unrolling finally states (either exception happened or premature return)
     curExcSym: PSym # Current exception
@@ -168,18 +160,18 @@ type
     nearestFinally: int # Index of the nearest finally block. For try/except it
                     # is their finally. For finally it is parent finally. Otherwise -1
     idgen: IdGenerator
+    varStates: Table[ItemId, int] # Used to detect if local variable belongs to multiple states
 
 const
   nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
             nkCommentStmt, nkMixinStmt, nkBindStmt} + procDefs
   emptyStateLabel = -1
+  localNotSeen = -1
+  localRequiresLifting = -2
 
 proc newStateAccess(ctx: var Ctx): PNode =
-  if ctx.stateVarSym.isNil:
-    result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)),
-        getStateField(ctx.g, ctx.fn), ctx.fn.info)
-  else:
-    result = newSymNode(ctx.stateVarSym)
+  result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)),
+      getStateField(ctx.g, ctx.fn), ctx.fn.info)
 
 proc newStateAssgn(ctx: var Ctx, toValue: PNode): PNode =
   # Creates state assignment:
@@ -195,24 +187,17 @@ proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
   result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.idgen, ctx.fn, ctx.fn.info)
   result.typ = typ
   result.flags.incl sfNoInit
-  assert(not typ.isNil)
+  assert(not typ.isNil, "Env var needs a type")
 
-  if not ctx.stateVarSym.isNil:
-    # We haven't gone through labmda lifting yet, so just create a local var,
-    # it will be lifted later
-    if ctx.tempVars.isNil:
-      ctx.tempVars = newNodeI(nkVarSection, ctx.fn.info)
-      addVar(ctx.tempVars, newSymNode(result))
-  else:
-    let envParam = getEnvParam(ctx.fn)
-    # let obj = envParam.typ.lastSon
-    result = addUniqueField(envParam.typ.elementType, result, ctx.g.cache, ctx.idgen)
+  let envParam = getEnvParam(ctx.fn)
+  # let obj = envParam.typ.lastSon
+  result = addUniqueField(envParam.typ.elementType, result, ctx.g.cache, ctx.idgen)
 
 proc newEnvVarAccess(ctx: Ctx, s: PSym): PNode =
-  if ctx.stateVarSym.isNil:
-    result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)), s, ctx.fn.info)
-  else:
-    result = newSymNode(s)
+  result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)), s, ctx.fn.info)
+
+proc newTempVarAccess(ctx: Ctx, s: PSym): PNode =
+  result = newSymNode(s, ctx.fn.info)
 
 proc newTmpResultAccess(ctx: var Ctx): PNode =
   if ctx.tmpResultSym.isNil:
@@ -255,9 +240,18 @@ proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   if result.len == 0 or result[^1].kind != nkGotoState:
     result.add(gotoOut)
 
-proc newTempVar(ctx: var Ctx, typ: PType): PSym =
-  result = ctx.newEnvVar(":tmpSlLower" & $ctx.tempVarId, typ)
+proc newTempVarDef(ctx: Ctx, s: PSym, initialValue: PNode): PNode =
+  var v = initialValue
+  if v == nil:
+    v = ctx.g.emptyNode
+  newTree(nkVarSection, newTree(nkIdentDefs, newSymNode(s), ctx.g.emptyNode, v))
+
+proc newTempVar(ctx: var Ctx, typ: PType, parent: PNode, initialValue: PNode = nil): PSym =
+  result = newSym(skVar, getIdent(ctx.g.cache, ":tmpSlLower" & $ctx.tempVarId), ctx.idgen, ctx.fn, ctx.fn.info)
   inc ctx.tempVarId
+  result.typ = typ
+  assert(not typ.isNil, "Temp var needs a type")
+  parent.add(ctx.newTempVarDef(result, initialValue))
 
 proc hasYields(n: PNode): bool =
   # TODO: This is very inefficient. It traverses the node, looking for nkYieldStmt.
@@ -429,21 +423,20 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
 
   result.res = n
 
-
-proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
+proc newTempVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
   if isEmptyType(v.typ):
     result = v
   else:
-    result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
+    result = newTree(nkFastAsgn, ctx.newTempVarAccess(s), v)
     result.info = v.info
 
 proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
   if input.kind == nkStmtListExpr:
     let (st, res) = exprToStmtList(input)
     output.add(st)
-    output.add(ctx.newEnvVarAsgn(sym, res))
+    output.add(ctx.newTempVarAsgn(sym, res))
   else:
-    output.add(ctx.newEnvVarAsgn(sym, input))
+    output.add(ctx.newTempVarAsgn(sym, input))
 
 proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode =
   result = newNodeI(nkStmtList, exprBody.info)
@@ -457,6 +450,12 @@ proc boolLit(g: ModuleGraph; info: TLineInfo; value: bool): PNode =
   result = newIntLit(g, info, ord value)
   result.typ = getSysType(g, info, tyBool)
 
+proc captureVar(c: var Ctx, s: PSym) =
+  if c.varStates.getOrDefault(s.itemId) != localRequiresLifting:
+    c.varStates[s.itemId] = localRequiresLifting # Mark this variable for lifting
+    let e = getEnvParam(c.fn)
+    discard addField(e.typ.elementType, s, c.g.cache, c.idgen)
+
 proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
   result = n
   case n.kind
@@ -513,9 +512,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       var tmp: PSym = nil
       let isExpr = not isEmptyType(n.typ)
       if isExpr:
-        tmp = ctx.newTempVar(n.typ)
         result = newNodeI(nkStmtListExpr, n.info)
         result.typ = n.typ
+        tmp = ctx.newTempVar(n.typ, result)
       else:
         result = newNodeI(nkStmtList, n.info)
 
@@ -566,7 +565,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         else:
           internalError(ctx.g.config, "lowerStmtListExpr(nkIf): " & $branch.kind)
 
-      if isExpr: result.add(ctx.newEnvVarAccess(tmp))
+      if isExpr: result.add(ctx.newTempVarAccess(tmp))
 
   of nkTryStmt, nkHiddenTryStmt:
     var ns = false
@@ -580,7 +579,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       if isExpr:
         result = newNodeI(nkStmtListExpr, n.info)
         result.typ = n.typ
-        let tmp = ctx.newTempVar(n.typ)
+        let tmp = ctx.newTempVar(n.typ, result)
 
         n[0] = ctx.convertExprBodyToAsgn(n[0], tmp)
         for i in 1..<n.len:
@@ -596,7 +595,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           else:
             internalError(ctx.g.config, "lowerStmtListExpr(nkTryStmt): " & $branch.kind)
         result.add(n)
-        result.add(ctx.newEnvVarAccess(tmp))
+        result.add(ctx.newTempVarAccess(tmp))
 
   of nkCaseStmt:
     var ns = false
@@ -609,9 +608,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       let isExpr = not isEmptyType(n.typ)
 
       if isExpr:
-        let tmp = ctx.newTempVar(n.typ)
         result = newNodeI(nkStmtListExpr, n.info)
         result.typ = n.typ
+        let tmp = ctx.newTempVar(n.typ, result)
 
         if n[0].kind == nkStmtListExpr:
           let (st, ex) = exprToStmtList(n[0])
@@ -628,7 +627,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           else:
             internalError(ctx.g.config, "lowerStmtListExpr(nkCaseStmt): " & $branch.kind)
         result.add(n)
-        result.add(ctx.newEnvVarAccess(tmp))
+        result.add(ctx.newTempVarAccess(tmp))
       elif n[0].kind == nkStmtListExpr:
         result = newNodeI(nkStmtList, n.info)
         let (st, ex) = exprToStmtList(n[0])
@@ -658,10 +657,10 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           result.add(st)
           cond = ex
 
-        let tmp = ctx.newTempVar(cond.typ)
-        result.add(ctx.newEnvVarAsgn(tmp, cond))
+        let tmp = ctx.newTempVar(cond.typ, result, cond)
+        # result.add(ctx.newTempVarAsgn(tmp, cond))
 
-        var check = ctx.newEnvVarAccess(tmp)
+        var check = ctx.newTempVarAccess(tmp)
         if n[0].sym.magic == mOr:
           check = ctx.g.newNotCall(check)
 
@@ -671,12 +670,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           let (st, ex) = exprToStmtList(cond)
           ifBody.add(st)
           cond = ex
-        ifBody.add(ctx.newEnvVarAsgn(tmp, cond))
+        ifBody.add(ctx.newTempVarAsgn(tmp, cond))
 
         let ifBranch = newTree(nkElifBranch, check, ifBody)
         let ifNode = newTree(nkIfStmt, ifBranch)
         result.add(ifNode)
-        result.add(ctx.newEnvVarAccess(tmp))
+        result.add(ctx.newTempVarAccess(tmp))
       else:
         for i in 0..<n.len:
           if n[i].kind == nkStmtListExpr:
@@ -685,9 +684,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
             n[i] = ex
 
           if n[i].kind in nkCallKinds: # XXX: This should better be some sort of side effect tracking
-            let tmp = ctx.newTempVar(n[i].typ)
-            result.add(ctx.newEnvVarAsgn(tmp, n[i]))
-            n[i] = ctx.newEnvVarAccess(tmp)
+            let tmp = ctx.newTempVar(n[i].typ, result, n[i])
+            # result.add(ctx.newTempVarAsgn(tmp, n[i]))
+            n[i] = ctx.newTempVarAccess(tmp)
 
         result.add(n)
 
@@ -703,6 +702,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         let (st, ex) = exprToStmtList(c[^1])
         result.add(st)
         c[^1] = ex
+      for i in 0 .. c.len - 3:
+        if c[i].kind == nkSym:
+          let s = c[i].sym
+          if sfForceLift in s.flags:
+            ctx.captureVar(s)
+
       result.add(varSect)
 
   of nkDiscardStmt, nkReturnStmt, nkRaiseStmt:
@@ -1279,13 +1284,6 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   result.info = n.info
 
   let localVars = newNodeI(nkStmtList, n.info)
-  if not ctx.stateVarSym.isNil:
-    let varSect = newNodeI(nkVarSection, n.info)
-    addVar(varSect, newSymNode(ctx.stateVarSym))
-    localVars.add(varSect)
-
-    if not ctx.tempVars.isNil:
-      localVars.add(ctx.tempVars)
 
   let blockStmt = newNodeI(nkBlockStmt, n.info)
   blockStmt.add(newSymNode(ctx.stateLoopLabel))
@@ -1433,15 +1431,67 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
 
+proc detectCapturedVars(c: var Ctx, n: PNode, stateIdx: int) =
+  case n.kind
+  of nkSym:
+    let s = n.sym
+    if s.kind in {skResult, skVar, skLet, skForVar, skTemp} and sfGlobal notin s.flags and s.owner == c.fn:
+      let vs = c.varStates.getOrDefault(s.itemId, localNotSeen)
+      if vs == localNotSeen: # First seing this variable
+        c.varStates[s.itemId] = stateIdx
+      elif vs == localRequiresLifting:
+        discard # Sym already marked
+      elif vs != stateIdx:
+        c.captureVar(s)
+  of nkReturnStmt:
+    if n[0].kind in {nkAsgn, nkFastAsgn, nkSinkAsgn}:
+      # we have a `result = result` expression produced by the closure
+      # transform, let's not touch the LHS in order to make the lifting pass
+      # correct when `result` is lifted
+      detectCapturedVars(c, n[0][1], stateIdx)
+    else:
+      detectCapturedVars(c, n[0], stateIdx)
+  else:
+    for i in 0 ..< n.safeLen:
+      detectCapturedVars(c, n[i], stateIdx)
+
+proc detectCapturedVars(c: var Ctx) =
+  for i, s in c.states:
+    detectCapturedVars(c, s.body, i)
+
+proc liftLocals(c: var Ctx, n: PNode): PNode =
+  result = n
+  case n.kind
+  of nkSym:
+    let s = n.sym
+    if c.varStates.getOrDefault(s.itemId) == localRequiresLifting:
+      # lift
+      let e = getEnvParam(c.fn)
+      let field = getFieldFromObj(e.typ.elementType, s)
+      assert(field != nil)
+      result = rawIndirectAccess(newSymNode(e), field, n.info)
+    # elif c.varStates.getOrDefault(s.itemId, localNotSeen) != localNotSeen:
+    #   echo "Not lifting ", s.name.s
+
+  of nkReturnStmt:
+    if n[0].kind in {nkAsgn, nkFastAsgn, nkSinkAsgn}:
+      # we have a `result = result` expression produced by the closure
+      # transform, let's not touch the LHS in order to make the lifting pass
+      # correct when `result` is lifted
+      n[0][1] = liftLocals(c, n[0][1])
+    else:
+      n[0] = liftLocals(c, n[0])
+  else:
+    for i in 0 ..< n.safeLen:
+      n[i] = liftLocals(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n: PNode): PNode =
   var ctx = Ctx(g: g, fn: fn, idgen: idgen)
 
-  if getEnvParam(fn).isNil:
-    # Lambda lifting was not done yet. Use temporary :state sym, which will
-    # be handled specially by lambda lifting. Local temp vars (if needed)
-    # should follow the same logic.
-    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), idgen, fn, fn.info)
-    ctx.stateVarSym.typ = g.createClosureIterStateType(fn, idgen)
+  # The transformation should always happen after at least partial lambdalifting
+  # is performed, so that the closure iter environment is always created upfront.
+  doAssert(getEnvParam(fn) != nil, "Env param not created before iter transformation")
+
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), idgen, fn, fn.info)
   var pc = PreprocessContext(finallys: @[], config: g.config, idgen: idgen)
   var n = preprocess(pc, n.toStmtList)
@@ -1466,6 +1516,10 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
   let caseDispatcher = newTreeI(nkCaseStmt, n.info,
       ctx.newStateAccess())
 
+  # Lamdalifting will not touch our locals, it is our responsibility to lift those that
+  # need it.
+  detectCapturedVars(ctx)
+
   for s in ctx.states:
     let body = ctx.transformStateAssignments(s.body)
     caseDispatcher.add newTreeI(nkOfBranch, body.info, g.newIntLit(body.info, s.label), body)
@@ -1473,6 +1527,7 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
   caseDispatcher.add newTreeI(nkElse, n.info, newTreeI(nkReturnStmt, n.info, g.emptyNode))
 
   result = wrapIntoStateLoop(ctx, caseDispatcher)
+  result = liftLocals(ctx, result)
 
   when false:
     echo "TRANSFORM TO STATES: "
@@ -1481,3 +1536,5 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
     echo "exception table:"
     for i, e in ctx.exceptionTable:
       echo i, " -> ", e
+
+    echo "ENV: ", renderTree(getEnvParam(fn).typ.elementType.n)

compiler/commands.nim

@@ -462,7 +462,6 @@ proc handleCmdInput*(conf: ConfigRef) =
 proc parseCommand*(command: string): Command =
   case command.normalize
   of "c", "cc", "compile", "compiletoc": cmdCompileToC
-  of "nir": cmdCompileToNir
   of "cpp", "compiletocpp": cmdCompileToCpp
   of "objc", "compiletooc": cmdCompileToOC
   of "js", "compiletojs": cmdCompileToJS
@@ -500,7 +499,6 @@ proc setCmd*(conf: ConfigRef, cmd: Command) =
   of cmdCompileToCpp: conf.backend = backendCpp
   of cmdCompileToOC: conf.backend = backendObjc
   of cmdCompileToJS: conf.backend = backendJs
-  of cmdCompileToNir: conf.backend = backendNir
   else: discard
 
 proc setCommandEarly*(conf: ConfigRef, command: string) =

compiler/condsyms.nim

@@ -166,3 +166,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasWarnStdPrefix")
 
   defineSymbol("nimHasVtables")
+  defineSymbol("nimHasJsNoLambdaLifting")

compiler/evaltempl.nim

@@ -56,6 +56,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
           #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
         if sfGenSym in s.flags:
+          # TODO: getIdent(c.ic, "`" & x.name.s & "`gensym" & $c.instID)
           result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $c.instID),
             if c.instLines: actual.info else: templ.info)
         else:

compiler/extccomp.nim

@@ -172,6 +172,22 @@ compiler vcc:
     cppXsupport: "",
     props: {hasCpp, hasAssume, hasDeclspec})
 
+# Nvidia CUDA NVCC Compiler
+compiler nvcc:
+  result = gcc()
+  result.name = "nvcc"
+  result.compilerExe = "nvcc"
+  result.cppCompiler = "nvcc"
+  result.compileTmpl = "-c -x cu -Xcompiler=\"$options\" $include -o $objfile $file"
+  result.linkTmpl = "$buildgui $builddll -o $exefile $objfiles -Xcompiler=\"$options\""
+
+# AMD HIPCC Compiler (rocm/cuda)
+compiler hipcc:
+  result = clang()
+  result.name = "hipcc"
+  result.compilerExe = "hipcc"
+  result.cppCompiler = "hipcc"
+
 compiler clangcl:
   result = vcc()
   result.name = "clang_cl"
@@ -285,7 +301,9 @@ const
     envcc(),
     icl(),
     icc(),
-    clangcl()]
+    clangcl(),
+    hipcc(),
+    nvcc()]
 
   hExt* = ".h"
 
@@ -319,7 +337,7 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
   var fullSuffix = suffix
   case conf.backend
   of backendCpp, backendJs, backendObjc: fullSuffix = "." & $conf.backend & suffix
-  of backendC, backendNir: discard
+  of backendC: discard
   of backendInvalid:
     # during parsing of cfg files; we don't know the backend yet, no point in
     # guessing wrong thing

compiler/ic/ic.nim

@@ -16,7 +16,7 @@ from std/os import removeFile, isAbsolute
 
 import ../../dist/checksums/src/checksums/sha1
 
-import ".." / nir / nirlineinfos
+import iclineinfos
 
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions, formatfloat]
@@ -59,8 +59,8 @@ type
     emittedTypeInfo*: seq[string]
     backendFlags*: set[ModuleBackendFlag]
 
-    syms*: seq[PackedSym]
-    types*: seq[PackedType]
+    syms*: OrderedTable[int32, PackedSym]
+    types*: OrderedTable[int32, PackedType]
     strings*: BiTable[string] # we could share these between modules.
     numbers*: BiTable[BiggestInt] # we also store floats in here so
                                   # that we can assure that every bit is kept
@@ -362,10 +362,10 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
   result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)
 
   if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item):
-    if t.uniqueId.item >= m.types.len:
-      setLen m.types, t.uniqueId.item+1
+    #if t.uniqueId.item >= m.types.len:
+    #  setLen m.types, t.uniqueId.item+1
 
-    var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
+    var p = PackedType(id: t.uniqueId.item, kind: t.kind, flags: t.flags, callConv: t.callConv,
       size: t.size, align: t.align, nonUniqueId: t.itemId.item,
       paddingAtEnd: t.paddingAtEnd)
     storeNode(p, t, n)
@@ -396,12 +396,12 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
   result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
 
   if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item):
-    if s.itemId.item >= m.syms.len:
-      setLen m.syms, s.itemId.item+1
+    #if s.itemId.item >= m.syms.len:
+    #  setLen m.syms, s.itemId.item+1
 
     assert sfForward notin s.flags
 
-    var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
+    var p = PackedSym(id: s.itemId.item, kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
       position: s.position, offset: s.offset, disamb: s.disamb, options: s.options,
       name: s.name.s.toLitId(m))
 
@@ -613,6 +613,10 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
     f.loadSection section
     f.loadSeq data
 
+  template loadTableSection(section, data) {.dirty.} =
+    f.loadSection section
+    f.loadOrderedTable data
+
   template loadTabSection(section, data) {.dirty.} =
     f.loadSection section
     f.load data
@@ -645,8 +649,8 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
     loadTabSection topLevelSection, m.topLevel
 
     loadTabSection bodiesSection, m.bodies
-    loadSeqSection symsSection, m.syms
-    loadSeqSection typesSection, m.types
+    loadTableSection symsSection, m.syms
+    loadTableSection typesSection, m.types
 
     loadSeqSection typeInstCacheSection, m.typeInstCache
     loadSeqSection procInstCacheSection, m.procInstCache
@@ -691,6 +695,10 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
     f.storeSection section
     f.store data
 
+  template storeTableSection(section, data) {.dirty.} =
+    f.storeSection section
+    f.storeOrderedTable data
+
   storeTabSection stringsSection, m.strings
 
   storeSeqSection checkSumsSection, m.includes
@@ -714,9 +722,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeTabSection topLevelSection, m.topLevel
 
   storeTabSection bodiesSection, m.bodies
-  storeSeqSection symsSection, m.syms
+  storeTableSection symsSection, m.syms
 
-  storeSeqSection typesSection, m.types
+  storeTableSection typesSection, m.types
 
   storeSeqSection typeInstCacheSection, m.typeInstCache
   storeSeqSection procInstCacheSection, m.procInstCache
@@ -767,8 +775,8 @@ type
     status*: ModuleStatus
     symsInit, typesInit, loadedButAliveSetChanged*: bool
     fromDisk*: PackedModule
-    syms: seq[PSym] # indexed by itemId
-    types: seq[PType]
+    syms: OrderedTable[int32, PSym] # indexed by itemId
+    types: OrderedTable[int32, PType]
     module*: PSym # the one true module symbol.
     iface, ifaceHidden: Table[PIdent, seq[PackedItemId]]
       # PackedItemId so that it works with reexported symbols too
@@ -961,11 +969,11 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s:
         loadToReplayNodes(g, c.config, c.cache, m, g[int m])
 
     assert g[si].status in {loaded, storing, stored}
-    if not g[si].symsInit:
-      g[si].symsInit = true
-      setLen g[si].syms, g[si].fromDisk.syms.len
+    #if not g[si].symsInit:
+    #  g[si].symsInit = true
+    #  setLen g[si].syms, g[si].fromDisk.syms.len
 
-    if g[si].syms[s.item] == nil:
+    if g[si].syms.getOrDefault(s.item) == nil:
       if g[si].fromDisk.syms[s.item].kind != skModule:
         result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item)
         # store it here early on, so that recursions work properly:
@@ -1012,11 +1020,11 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
     assert g[si].status in {loaded, storing, stored}
     assert t.item > 0
 
-    if not g[si].typesInit:
-      g[si].typesInit = true
-      setLen g[si].types, g[si].fromDisk.types.len
+    #if not g[si].typesInit:
+    #  g[si].typesInit = true
+    #  setLen g[si].types, g[si].fromDisk.types.len
 
-    if g[si].types[t.item] == nil:
+    if g[si].types.getOrDefault(t.item) == nil:
       result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item)
       # store it here early on, so that recursions work properly:
       g[si].types[t.item] = result
@@ -1155,10 +1163,7 @@ proc loadProcBody*(config: ConfigRef, cache: IdentCache;
 proc loadTypeFromId*(config: ConfigRef, cache: IdentCache;
                      g: var PackedModuleGraph; module: int; id: PackedItemId): PType =
   bench g.loadType:
-    if id.item < g[module].types.len:
-      result = g[module].types[id.item]
-    else:
-      result = nil
+    result = g[module].types.getOrDefault(id.item)
     if result == nil:
       var decoder = PackedDecoder(
         lastModule: int32(-1),
@@ -1171,10 +1176,7 @@ proc loadTypeFromId*(config: ConfigRef, cache: IdentCache;
 proc loadSymFromId*(config: ConfigRef, cache: IdentCache;
                     g: var PackedModuleGraph; module: int; id: PackedItemId): PSym =
   bench g.loadSym:
-    if id.item < g[module].syms.len:
-      result = g[module].syms[id.item]
-    else:
-      result = nil
+    result = g[module].syms.getOrDefault(id.item)
     if result == nil:
       var decoder = PackedDecoder(
         lastModule: int32(-1),
@@ -1190,19 +1192,6 @@ proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; confi
   else:
     ItemId(module: toFileIndex(id.module, g[thisModule].fromDisk, config).int32, item: id.item)
 
-proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) =
-  var bugs = 0
-  for i in 1 .. high(m.syms):
-    if m.syms[i].kind == skUnknown:
-      echo "EMPTY ID ", i, " module ", moduleId, " ", toFullPath(config, FileIndex(moduleId))
-      inc bugs
-  assert bugs == 0
-  when false:
-    var nones = 0
-    for i in 1 .. high(m.types):
-      inc nones, m.types[i].kind == tyNone
-    assert nones < 1
-
 proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                            moduleSym: PSym; m: PackedModule) =
   # For now only used for heavy debugging. In the future we could use this to reduce the

compiler/ic/iclineinfos.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
+#        (c) Copyright 2024 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.

compiler/ic/integrity.nim

@@ -10,7 +10,7 @@
 ## Integrity checking for a set of .rod files.
 ## The set must cover a complete Nim project.
 
-import std/sets
+import std/[sets, tables]
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -108,18 +108,18 @@ proc checkModule(c: var CheckedContext; m: PackedModule) =
   # We check that:
   # - Every symbol references existing types and symbols.
   # - Every tree node references existing types and symbols.
-  for i in 0..high(m.syms):
-    checkLocalSym c, int32(i)
+  for _, v in pairs(m.syms):
+    checkLocalSym c, v.id
 
   checkTree c, m.toReplay
   checkTree c, m.topLevel
 
   for e in m.exports:
-    assert e[1] >= 0 and e[1] < m.syms.len
+    #assert e[1] >= 0 and e[1] < m.syms.len
     assert e[0] == m.syms[e[1]].name
 
   for e in m.compilerProcs:
-    assert e[1] >= 0 and e[1] < m.syms.len
+    #assert e[1] >= 0 and e[1] < m.syms.len
     assert e[0] == m.syms[e[1]].name
 
   checkLocalSymIds c, m, m.converters

compiler/ic/navigator.nim

@@ -11,7 +11,7 @@
 ## IDE-like features. It uses the set of .rod files to accomplish
 ## its task. The set must cover a complete Nim project.
 
-import std/sets
+import std/[sets, tables]
 
 from std/os import nil
 from std/private/miscdollars import toLocation
@@ -20,7 +20,7 @@ when defined(nimPreviewSlimSystem):
   import std/assertions
 
 import ".." / [ast, modulegraphs, msgs, options]
-import ".." / nir / nirlineinfos
+import iclineinfos
 import packed_ast, bitabs, ic
 
 type

compiler/ic/packed_ast.nim

@@ -16,7 +16,7 @@ import std/[hashes, tables, strtabs]
 import bitabs, rodfiles
 import ".." / [ast, options]
 
-import ".." / nir / nirlineinfos
+import iclineinfos
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -47,6 +47,7 @@ type
     path*: NodeId
 
   PackedSym* = object
+    id*: int32
     kind*: TSymKind
     name*: LitId
     typ*: PackedItemId
@@ -71,6 +72,7 @@ type
     instantiatedFrom*: PackedItemId
 
   PackedType* = object
+    id*: int32
     kind*: TTypeKind
     callConv*: TCallingConvention
     #nodekind*: TNodeKind

compiler/ic/rodfiles.nim

@@ -19,6 +19,8 @@ from std/typetraits import supportsCopyMem
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions]
 
+import std / tables
+
 ## Overview
 ## ========
 ## `RodFile` represents a Rod File (versioned binary format), and the
@@ -170,6 +172,18 @@ proc storeSeq*[T](f: var RodFile; s: seq[T]) =
     for i in 0..<s.len:
       storePrim(f, s[i])
 
+proc storeOrderedTable*[K, T](f: var RodFile; s: OrderedTable[K, T]) =
+  if f.err != ok: return
+  if s.len >= high(int32):
+    setError f, tooBig
+    return
+  var lenPrefix = int32(s.len)
+  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    setError f, ioFailure
+  else:
+    for _, v in s:
+      storePrim(f, v)
+
 proc loadPrim*(f: var RodFile; s: var string) =
   ## Read a string, the length was stored as a prefix
   if f.err != ok: return
@@ -211,6 +225,19 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
     for i in 0..<lenPrefix:
       loadPrim(f, s[i])
 
+proc loadOrderedTable*[K, T](f: var RodFile; s: var OrderedTable[K, T]) =
+  ## `T` must be compatible with `copyMem`, see `loadPrim`
+  if f.err != ok: return
+  var lenPrefix = int32(0)
+  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    setError f, ioFailure
+  else:
+    s = initOrderedTable[K, T](lenPrefix)
+    for i in 0..<lenPrefix:
+      var x = default T
+      loadPrim(f, x)
+      s[x.id] = x
+
 proc storeHeader*(f: var RodFile; cookie = defaultCookie) =
   ## stores the header which is described by `cookie`.
   if f.err != ok: return

compiler/injectdestructors.nim

@@ -653,7 +653,7 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false,
       for j in 0 ..< it.len-1:
         branch[j] = copyTree(it[j])
       var ofScope = nestedScope(s, it.lastSon)
-      branch[^1] = if it[^1].typ.isEmptyType or willProduceStmt:
+      branch[^1] = if n.typ.isEmptyType or it[^1].typ.isEmptyType or willProduceStmt:
                      processScope(c, ofScope, maybeVoid(it[^1], ofScope))
                    else:
                      processScopeExpr(c, ofScope, it[^1], processCall, tmpFlags)
@@ -701,7 +701,7 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false,
         #Condition needs to be destroyed outside of the condition/branch scope
         branch[0] = p(it[0], c, s, normal)
 
-      branch[^1] = if it[^1].typ.isEmptyType or willProduceStmt:
+      branch[^1] = if n.typ.isEmptyType or it[^1].typ.isEmptyType or willProduceStmt:
                      processScope(c, branchScope, maybeVoid(it[^1], branchScope))
                    else:
                      processScopeExpr(c, branchScope, it[^1], processCall, tmpFlags)
@@ -872,7 +872,8 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
       for i in 1..<n.len:
         if n[i].kind == nkExprColonExpr:
           let field = lookupFieldAgain(t, n[i][0].sym)
-          if field != nil and sfCursor in field.flags:
+          if field != nil and (sfCursor in field.flags or field.typ.kind in {tyOpenArray, tyVarargs}):
+            # don't sink fields with openarray types
             result[i][1] = p(n[i][1], c, s, normal)
           else:
             result[i][1] = p(n[i][1], c, s, m)

compiler/installer.ini

@@ -6,7 +6,7 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64;loongarch64
   macosx: i386;amd64;powerpc64;arm64
   solaris: i386;amd64;sparc;sparc64
   freebsd: i386;amd64;powerpc64;arm;arm64;riscv64;sparc64;mips;mipsel;mips64;mips64el;powerpc;powerpc64el

compiler/jsgen.nim

@@ -111,13 +111,25 @@ type
     blocks: seq[TBlock]
     extraIndent: int
     previousFileName: string  # For frameInfo inside templates.
+    # legacy: generatedParamCopies and up fields are used for jsNoLambdaLifting
+    generatedParamCopies: IntSet
+    up: PProc     # up the call chain; required for closure support
 
 template config*(p: PProc): ConfigRef = p.module.config
 
 proc indentLine(p: PProc, r: Rope): Rope =
   var p = p
-  let ind = p.blocks.len + p.extraIndent
-  result = repeat(' ', ind*2) & r
+  if jsNoLambdaLifting in p.config.legacyFeatures:
+    var ind = 0
+    while true:
+      inc ind, p.blocks.len + p.extraIndent
+      if p.up == nil or p.up.prc != p.prc.owner:
+        break
+      p = p.up
+    result = repeat(' ', ind*2) & r
+  else:
+    let ind = p.blocks.len + p.extraIndent
+    result = repeat(' ', ind*2) & r
 
 template line(p: PProc, added: string) =
   p.body.add(indentLine(p, rope(added)))
@@ -1200,12 +1212,13 @@ proc genIf(p: PProc, n: PNode, r: var TCompRes) =
 proc generateHeader(p: PProc, prc: PSym): Rope =
   result = ""
   let typ = prc.typ
-  if typ.callConv == ccClosure:
-    # we treat Env as the `this` parameter of the function
-    # to keep it simple
-    let env = prc.ast[paramsPos].lastSon
-    assert env.kind == nkSym, "env is missing"
-    env.sym.loc.r = "this"
+  if jsNoLambdaLifting notin p.config.legacyFeatures:
+    if typ.callConv == ccClosure:
+      # we treat Env as the `this` parameter of the function
+      # to keep it simple
+      let env = prc.ast[paramsPos].lastSon
+      assert env.kind == nkSym, "env is missing"
+      env.sym.loc.r = "this"
 
   for i in 1..<typ.n.len:
     assert(typ.n[i].kind == nkSym)
@@ -1239,7 +1252,8 @@ const
 
 proc needsNoCopy(p: PProc; y: PNode): bool =
   return y.kind in nodeKindsNeedNoCopy or
-        ((mapType(y.typ) != etyBaseIndex) and
+        ((mapType(y.typ) != etyBaseIndex or
+          (jsNoLambdaLifting in p.config.legacyFeatures and y.kind == nkSym and y.sym.kind == skParam)) and
           (skipTypes(y.typ, abstractInst).kind in
             {tyRef, tyPtr, tyLent, tyVar, tyCstring, tyProc, tyOwned, tyOpenArray} + IntegralTypes))
 
@@ -1268,7 +1282,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
       lineF(p, "$1 = nimCopy(null, $2, $3);$n",
                [a.rdLoc, b.res, genTypeInfo(p, y.typ)])
   of etyObject:
-    if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind in {tyVar, tyLent, tyOpenArray, tyVarargs} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
@@ -1590,7 +1604,30 @@ proc attachProc(p: PProc; s: PSym) =
 
 proc genProcForSymIfNeeded(p: PProc, s: PSym) =
   if not p.g.generatedSyms.containsOrIncl(s.id):
-    attachProc(p, s)
+    if jsNoLambdaLifting in p.config.legacyFeatures:
+      let newp = genProc(p, s)
+      var owner = p
+      while owner != nil and owner.prc != s.owner:
+        owner = owner.up
+      if owner != nil: owner.locals.add(newp)
+      else: attachProc(p, newp, s)
+    else:
+      attachProc(p, s)
+
+proc genCopyForParamIfNeeded(p: PProc, n: PNode) =
+  let s = n.sym
+  if p.prc == s.owner or needsNoCopy(p, n):
+    return
+  var owner = p.up
+  while true:
+    if owner == nil:
+      internalError(p.config, n.info, "couldn't find the owner proc of the closed over param: " & s.name.s)
+    if owner.prc == s.owner:
+      if not owner.generatedParamCopies.containsOrIncl(s.id):
+        let copy = "$1 = nimCopy(null, $1, $2);$n" % [s.loc.r, genTypeInfo(p, s.typ)]
+        owner.locals.add(owner.indentLine(copy))
+      return
+    owner = owner.up
 
 proc genVarInit(p: PProc, v: PSym, n: PNode)
 
@@ -1602,6 +1639,8 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
     if sfCompileTime in s.flags:
       genVarInit(p, s, if s.astdef != nil: s.astdef else: newNodeI(nkEmpty, s.info))
+    if jsNoLambdaLifting in p.config.legacyFeatures and s.kind == skParam:
+      genCopyForParamIfNeeded(p, n)
     let k = mapType(p, s.typ)
     if k == etyBaseIndex:
       r.typ = etyBaseIndex
@@ -2018,7 +2057,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     gen(p, n, a)
     case mapType(p, v.typ)
     of etyObject, etySeq:
-      if needsNoCopy(p, n):
+      if v.typ.kind in {tyOpenArray, tyVarargs} or needsNoCopy(p, n):
         s = a.res
       else:
         useMagic(p, "nimCopy")
@@ -2688,6 +2727,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
   #if gVerbosity >= 3:
   #  echo "BEGIN generating code for: " & prc.name.s
   var p = newProc(oldProc.g, oldProc.module, prc.ast, prc.options)
+  p.up = oldProc
   var returnStmt: Rope = ""
   var resultAsgn: Rope = ""
   var name = mangleName(p.module, prc)
@@ -2911,14 +2951,17 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     else:
       genCall(p, n, r)
   of nkClosure:
-    let tmp = getTemp(p)
-    var a: TCompRes = default(TCompRes)
-    var b: TCompRes = default(TCompRes)
-    gen(p, n[0], a)
-    gen(p, n[1], b)
-    lineF(p, "$1 = $2.bind($3); $1.ClP_0 = $2; $1.ClE_0 = $3;$n", [tmp, a.rdLoc, b.rdLoc])
-    r.res = tmp
-    r.kind = resVal
+    if jsNoLambdaLifting in p.config.legacyFeatures:
+      gen(p, n[0], r)
+    else:
+      let tmp = getTemp(p)
+      var a: TCompRes = default(TCompRes)
+      var b: TCompRes = default(TCompRes)
+      gen(p, n[0], a)
+      gen(p, n[1], b)
+      lineF(p, "$1 = $2.bind($3); $1.ClP_0 = $2; $1.ClE_0 = $3;$n", [tmp, a.rdLoc, b.rdLoc])
+      r.res = tmp
+      r.kind = resVal
   of nkCurly: genSetConstr(p, n, r)
   of nkBracket: genArrayConstr(p, n, r)
   of nkPar, nkTupleConstr: genTupleConstr(p, n, r)

compiler/lambdalifting.nim

@@ -145,11 +145,13 @@ proc createStateField(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym =
   result = newSym(skField, getIdent(g.cache, ":state"), idgen, iter, iter.info)
   result.typ = createClosureIterStateType(g, iter, idgen)
 
+template isIterator*(owner: PSym): bool =
+  owner.kind == skIterator and owner.typ.callConv == ccClosure
+
 proc createEnvObj(g: ModuleGraph; idgen: IdGenerator; owner: PSym; info: TLineInfo): PType =
-  # YYY meh, just add the state field for every closure for now, it's too
-  # hard to figure out if it comes from a closure iterator:
   result = createObj(g, idgen, owner, info, final=false)
-  rawAddField(result, createStateField(g, owner, idgen))
+  if owner.isIterator:
+    rawAddField(result, createStateField(g, owner, idgen))
 
 proc getClosureIterResult*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym =
   if resultPos < iter.ast.len:
@@ -172,26 +174,22 @@ proc addHiddenParam(routine: PSym, param: PSym) =
   assert sfFromGeneric in param.flags
   #echo "produced environment: ", param.id, " for ", routine.id
 
-proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
+proc getEnvParam*(routine: PSym): PSym =
   let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName:
     result = hidden.sym
     assert sfFromGeneric in result.flags
   else:
+    result = nil
+
+proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
+  result = getEnvParam(routine)
+  if result.isNil:
     # writeStackTrace()
     localError(g.config, routine.info, "internal error: could not find env param for " & routine.name.s)
     result = routine
 
-proc getEnvParam*(routine: PSym): PSym =
-  let params = routine.ast[paramsPos]
-  let hidden = lastSon(params)
-  if hidden.kind == nkSym and hidden.sym.name.s == paramName:
-    result = hidden.sym
-    assert sfFromGeneric in result.flags
-  else:
-    result = nil
-
 proc interestingVar(s: PSym): bool {.inline.} =
   result = s.kind in {skVar, skLet, skTemp, skForVar, skParam, skResult} and
     sfGlobal notin s.flags and
@@ -230,14 +228,10 @@ proc makeClosure*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; env: PNode; inf
   if tfHasAsgn in result.typ.flags or optSeqDestructors in g.config.globalOptions:
     prc.flags.incl sfInjectDestructors
 
-proc interestingIterVar(s: PSym): bool {.inline.} =
-  # XXX optimization: Only lift the variable if it lives across
-  # yield/return boundaries! This can potentially speed up
-  # closure iterators quite a bit.
-  result = s.kind in {skResult, skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags
-
-template isIterator*(owner: PSym): bool =
-  owner.kind == skIterator and owner.typ.callConv == ccClosure
+template liftingHarmful(conf: ConfigRef; owner: PSym): bool =
+  ## lambda lifting can be harmful for JS-like code generators.
+  let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
+  jsNoLambdaLifting in conf.legacyFeatures and conf.backend == backendJs and not isCompileTime
 
 proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; idgen: IdGenerator; owner: PSym) =
   if owner.kind != skMacro:
@@ -255,6 +249,7 @@ proc genCreateEnv(env: PNode): PNode =
 
 proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
+  if liftingHarmful(g.config, owner): return n
   let iter = n.sym
   assert iter.isIterator
 
@@ -279,15 +274,6 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN
   createTypeBoundOpsLL(g, env.typ, n.info, idgen, owner)
   result.add makeClosure(g, idgen, iter, env, n.info)
 
-proc freshVarForClosureIter*(g: ModuleGraph; s: PSym; idgen: IdGenerator; owner: PSym): PNode =
-  let envParam = getHiddenParam(g, owner)
-  let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr})
-  let field = addField(obj, s, g.cache, idgen)
-
-  var access = newSymNode(envParam)
-  assert obj.kind == tyObject
-  result = rawIndirectAccess(access, field, s.info)
-
 # ------------------ new stuff -------------------------------------------
 
 proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
@@ -335,9 +321,13 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
                         info: TLineInfo): PType =
   result = c.ownerToType.getOrDefault(owner.id)
   if result.isNil:
-    result = newType(tyRef, c.idgen, owner)
-    let obj = createEnvObj(c.graph, c.idgen, owner, info)
-    rawAddSon(result, obj)
+    let env = getEnvParam(owner)
+    if env.isNil or not owner.isIterator:
+      result = newType(tyRef, c.idgen, owner)
+      let obj = createEnvObj(c.graph, c.idgen, owner, info)
+      rawAddSon(result, obj)
+    else:
+      result = env.typ
     c.ownerToType[owner.id] = result
 
 proc asOwnedRef(c: var DetectionPass; t: PType): PType =
@@ -453,17 +443,6 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       if owner.isIterator:
         c.somethingToDo = true
         addClosureParam(c, owner, n.info)
-        if interestingIterVar(s):
-          if not c.capturedVars.contains(s.id):
-            if not c.inTypeOf: c.capturedVars.incl(s.id)
-            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
-            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef, tyPtr})
-
-            if s.name.id == getIdent(c.graph.cache, ":state").id:
-              obj.n[0].sym.flags.incl sfNoInit
-              obj.n[0].sym.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
-            else:
-              discard addField(obj, s, c.graph.cache, c.idgen)
     # direct or indirect dependency:
     elif innerClosure or interested:
       discard """
@@ -767,8 +746,6 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
     elif s.id in d.capturedVars:
       if s.owner != owner:
         result = accessViaEnvParam(d.graph, n, owner)
-      elif owner.isIterator and interestingIterVar(s):
-        result = accessViaEnvParam(d.graph, n, owner)
       else:
         result = accessViaEnvVar(n, owner, d, c)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
@@ -879,12 +856,16 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
                   idgen: IdGenerator; flags: TransformFlags): PNode =
   let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro
 
-  if body.kind == nkEmpty or
+  if body.kind == nkEmpty or (jsNoLambdaLifting in g.config.legacyFeatures and
+      g.config.backend == backendJs and not isCompileTime) or
       (fn.skipGenericOwner.kind != skModule and force notin flags):
 
     # ignore forward declaration:
     result = body
     tooEarly = true
+    if fn.isIterator:
+      var d = initDetectionPass(g, fn, idgen)
+      addClosureParam(d, fn, body.info)
   else:
     var d = initDetectionPass(g, fn, idgen)
     detectCapturedVars(body, fn, d)
@@ -939,6 +920,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
           break
         ...
     """
+  if liftingHarmful(g.config, owner): return body
   if not (body.kind == nkForStmt and body[^2].kind in nkCallKinds):
     localError(g.config, body.info, "ignored invalid for loop")
     return body

compiler/liftdestructors.nim

@@ -157,7 +157,7 @@ proc genWasMovedCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
   result.add(newSymNode(op))
   result.add genAddr(c, x)
 
-proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
+proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool, enforceWasMoved = false) =
   case n.kind
   of nkSym:
     if c.filterDiscriminator != nil: return
@@ -167,6 +167,8 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
         enforceDefaultOp:
       defaultOp(c, f.typ, body, x.dotField(f), b)
     else:
+      if enforceWasMoved:
+        body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x.dotField(f))
       fillBody(c, f.typ, body, x.dotField(f), b)
   of nkNilLit: discard
   of nkRecCase:
@@ -205,7 +207,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       branch[^1] = newNodeI(nkStmtList, c.info)
 
       fillBodyObj(c, n[i].lastSon, branch[^1], x, y,
-                  enforceDefaultOp = localEnforceDefaultOp)
+                  enforceDefaultOp = localEnforceDefaultOp, enforceWasMoved = c.kind == attachedAsgn)
       if branch[^1].len == 0: inc emptyBranches
       caseStmt.add(branch)
     if emptyBranches != n.len-1:
@@ -216,7 +218,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       fillBodyObj(c, n[0], body, x, y, enforceDefaultOp = false)
     c.filterDiscriminator = oldfilterDiscriminator
   of nkRecList:
-    for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp)
+    for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp, enforceWasMoved)
   else:
     illFormedAstLocal(n, c.g.config)
 
@@ -282,6 +284,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     c.kind = attachedDestructor
     fillBodyObjTImpl(c, t, body, blob, y)
     c.kind = prevKind
+
   else:
     fillBodyObjTImpl(c, t, body, x, y)
 

compiler/lineinfos.nim

@@ -95,6 +95,7 @@ type
     warnGenericsIgnoredInjection = "GenericsIgnoredInjection",
     warnStdPrefix = "StdPrefix"
     warnUser = "User",
+    warnGlobalVarConstructorTemporary = "GlobalVarConstructorTemporary",
     # hints
     hintSuccess = "Success", hintSuccessX = "SuccessX",
     hintCC = "CC",
@@ -200,6 +201,7 @@ const
     warnGenericsIgnoredInjection: "$1",
     warnStdPrefix: "$1 needs the 'std' prefix",
     warnUser: "$1",
+    warnGlobalVarConstructorTemporary: "global variable '$1' initialization requires a temporary variable",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
     hintSuccessX: "$build\n$loc lines; ${sec}s; $mem; proj: $project; out: $output",

compiler/main.nim

@@ -22,8 +22,6 @@ import
   modules,
   modulegraphs, lineinfos, pathutils, vmprofiler
 
-# ensure NIR compiles:
-import nir / nir
 
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions]
@@ -48,9 +46,6 @@ proc writeDepsFile(g: ModuleGraph) =
       f.writeLine(toFullPath(g.config, k))
   f.close()
 
-proc writeNinjaFile(g: ModuleGraph) =
-  discard "to implement"
-
 proc writeCMakeDepsFile(conf: ConfigRef) =
   ## write a list of C files for build systems like CMake.
   ## only updated when the C file list changes.
@@ -161,26 +156,6 @@ proc commandCompileToC(graph: ModuleGraph) =
     if optGenCDeps in graph.config.globalOptions:
       writeCMakeDepsFile(conf)
 
-proc commandCompileToNir(graph: ModuleGraph) =
-  let conf = graph.config
-  extccomp.initVars(conf)
-  if conf.symbolFiles == disabledSf:
-    if {optRun, optForceFullMake} * conf.globalOptions == {optRun}:
-      if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile):
-        # nothing changed
-        graph.config.notes = graph.config.mainPackageNotes
-        return
-
-  if not extccomp.ccHasSaneOverflow(conf):
-    conf.symbols.defineSymbol("nimEmulateOverflowChecks")
-
-  if conf.symbolFiles == disabledSf:
-    setPipeLinePass(graph, NirPass)
-  else:
-    setPipeLinePass(graph, SemPass)
-  compilePipelineProject(graph)
-  writeNinjaFile(graph)
-
 proc commandJsonScript(graph: ModuleGraph) =
   extccomp.runJsonBuildInstructions(graph.config, graph.config.jsonBuildInstructionsFile)
 
@@ -197,16 +172,13 @@ proc commandCompileToJS(graph: ModuleGraph) =
     if optGenScript in conf.globalOptions:
       writeDepsFile(graph)
 
-proc commandInteractive(graph: ModuleGraph; useNir: bool) =
+proc commandInteractive(graph: ModuleGraph) =
   graph.config.setErrorMaxHighMaybe
   initDefines(graph.config.symbols)
-  if useNir:
-    defineSymbol(graph.config.symbols, "noSignalHandler")
-  else:
-    defineSymbol(graph.config.symbols, "nimscript")
+  defineSymbol(graph.config.symbols, "nimscript")
   # note: seems redundant with -d:nimHasLibFFI
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
-  setPipeLinePass(graph, if useNir: NirReplPass else: InterpreterPass)
+  setPipeLinePass(graph, InterpreterPass)
   compilePipelineSystemModule(graph)
   if graph.config.commandArgs.len > 0:
     discard graph.compilePipelineModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
@@ -293,8 +265,6 @@ proc mainCommand*(graph: ModuleGraph) =
         # and it has added this define implictly, so we must undo that here.
         # A better solution might be to fix system.nim
         undefSymbol(conf.symbols, "useNimRtl")
-    of backendNir:
-      if conf.exc == excNone: conf.exc = excGoto
     of backendInvalid: raiseAssert "unreachable"
 
   proc compileToBackend() =
@@ -305,7 +275,6 @@ proc mainCommand*(graph: ModuleGraph) =
     of backendCpp: commandCompileToC(graph)
     of backendObjc: commandCompileToC(graph)
     of backendJs: commandCompileToJS(graph)
-    of backendNir: commandCompileToNir(graph)
     of backendInvalid: raiseAssert "unreachable"
 
   template docLikeCmd(body) =
@@ -444,7 +413,7 @@ proc mainCommand*(graph: ModuleGraph) =
     wantMainModule(conf)
     commandView(graph)
     #msgWriteln(conf, "Beware: Indentation tokens depend on the parser's state!")
-  of cmdInteractive: commandInteractive(graph, isDefined(conf, "nir"))
+  of cmdInteractive: commandInteractive(graph)
   of cmdNimscript:
     if conf.projectIsCmd or conf.projectIsStdin: discard
     elif not fileExists(conf.projectFull):

compiler/modulegraphs.nim

@@ -62,8 +62,6 @@ type
     CgenPass
     EvalPass
     InterpreterPass
-    NirPass
-    NirReplPass
     GenDependPass
     Docgen2TexPass
     Docgen2JsonPass

compiler/nim.nim

@@ -116,8 +116,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
     conf.backend = backendC
 
   if conf.selectedGC == gcUnselected:
-    if conf.backend in {backendC, backendCpp, backendObjc, backendNir} or
-        (conf.cmd == cmdInteractive and isDefined(conf, "nir")) or
+    if conf.backend in {backendC, backendCpp, backendObjc} or
         (conf.cmd in cmdDocLike and conf.backend != backendJs):
       initOrcDefines(conf)
 

compiler/nir/cir.nim

@@ -1,983 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-# We produce C code as a list of tokens.
-
-import std / [assertions, syncio, tables, intsets, formatfloat]
-from std / strutils import toOctal
-import .. / ic / [bitabs, rodfiles]
-import nirtypes, nirinsts, nirfiles
-import ../../dist/checksums/src/checksums/md5
-
-type
-  Token = LitId # indexing into the tokens BiTable[string]
-
-  PredefinedToken = enum
-    IgnoreMe = "<unused>"
-    EmptyToken = ""
-    CurlyLe = "{"
-    CurlyRi = "}"
-    ParLe = "("
-    ParRi = ")"
-    BracketLe = "["
-    BracketRi = "]"
-    NewLine = "\n"
-    Semicolon = ";"
-    Comma = ", "
-    Space = " "
-    Colon = ": "
-    Dot = "."
-    Arrow = "->"
-    Star = "*"
-    Amp = "&"
-    AsgnOpr = " = "
-    ScopeOpr = "::"
-    ConstKeyword = "const "
-    StaticKeyword = "static "
-    ExternKeyword = "extern "
-    WhileKeyword = "while "
-    IfKeyword = "if ("
-    ElseKeyword = "else "
-    SwitchKeyword = "switch "
-    CaseKeyword = "case "
-    DefaultKeyword = "default:"
-    BreakKeyword = "break"
-    NullPtr = "nullptr"
-    IfNot = "if (!("
-    ReturnKeyword = "return "
-    TypedefStruct = "typedef struct "
-    TypedefUnion = "typedef union "
-    IncludeKeyword = "#include "
-
-proc fillTokenTable(tab: var BiTable[string]) =
-  for e in EmptyToken..high(PredefinedToken):
-    let id = tab.getOrIncl $e
-    assert id == LitId(e), $(id, " ", ord(e))
-
-type
-  GeneratedCode* = object
-    m: NirModule
-    includes: seq[LitId]
-    includedHeaders: IntSet
-    data: seq[LitId]
-    protos: seq[LitId]
-    code: seq[LitId]
-    init: seq[LitId]
-    tokens: BiTable[string]
-    emittedStrings: IntSet
-    needsPrefix: IntSet
-    generatedTypes: IntSet
-    mangledModules: Table[LitId, LitId]
-
-proc initGeneratedCode*(m: sink NirModule): GeneratedCode =
-  result = GeneratedCode(m: m, code: @[], tokens: initBiTable[string]())
-  fillTokenTable(result.tokens)
-
-proc add*(g: var GeneratedCode; t: PredefinedToken) {.inline.} =
-  g.code.add Token(t)
-
-proc add*(g: var GeneratedCode; s: string) {.inline.} =
-  g.code.add g.tokens.getOrIncl(s)
-
-proc mangleModuleName(c: var GeneratedCode; key: LitId): LitId =
-  result = c.mangledModules.getOrDefault(key, LitId(0))
-  if result == LitId(0):
-    let u {.cursor.} = c.m.lit.strings[key]
-    var last = u.len - len(".nim") - 1
-    var start = last
-    while start >= 0 and u[start] != '/': dec start
-    var sum = getMD5(u)
-    sum.setLen(8)
-    let dest = u.substr(start+1, last) & sum
-    result = c.tokens.getOrIncl(dest)
-    c.mangledModules[key] = result
-
-type
-  CppFile = object
-    f: File
-
-proc write(f: var CppFile; s: string) = write(f.f, s)
-proc write(f: var CppFile; c: char) = write(f.f, c)
-
-proc writeTokenSeq(f: var CppFile; s: seq[Token]; c: GeneratedCode) =
-  var indent = 0
-  for i in 0..<s.len:
-    let x = s[i]
-    case x
-    of Token(CurlyLe):
-      inc indent
-      write f, c.tokens[x]
-      write f, "\n"
-      for i in 1..indent*2: write f, ' '
-    of Token(CurlyRi):
-      dec indent
-      write f, c.tokens[x]
-      if i+1 < s.len and s[i+1] == Token(CurlyRi):
-        discard
-      else:
-        write f, "\n"
-        for i in 1..indent*2: write f, ' '
-    of Token(Semicolon):
-      write f, c.tokens[x]
-      if i+1 < s.len and s[i+1] == Token(CurlyRi):
-        discard "no newline before }"
-      else:
-        write f, "\n"
-        for i in 1..indent*2: write f, ' '
-    of Token(NewLine):
-      write f, c.tokens[x]
-      for i in 1..indent*2: write f, ' '
-    else:
-      write f, c.tokens[x]
-
-
-# Type graph
-
-type
-  TypeList = object
-    processed: IntSet
-    s: seq[(TypeId, PredefinedToken)]
-
-proc add(dest: var TypeList; elem: TypeId; decl: PredefinedToken) =
-  if not containsOrIncl(dest.processed, int(elem)):
-    dest.s.add (elem, decl)
-
-type
-  TypeOrder = object
-    forwardedDecls, ordered: TypeList
-    typeImpls: Table[string, TypeId]
-    lookedAt: IntSet
-
-proc traverseObject(types: TypeGraph; lit: Literals; c: var TypeOrder; t: TypeId)
-
-proc recordDependency(types: TypeGraph; lit: Literals; c: var TypeOrder; parent, child: TypeId) =
-  var ch = child
-  var viaPointer = false
-  while true:
-    case types[ch].kind
-    of APtrTy, UPtrTy, AArrayPtrTy, UArrayPtrTy:
-      viaPointer = true
-      ch = elementType(types, ch)
-    of LastArrayTy:
-      ch = elementType(types, ch)
-    else:
-      break
-
-  case types[ch].kind
-  of ObjectTy, UnionTy:
-    let decl = if types[ch].kind == ObjectTy: TypedefStruct else: TypedefUnion
-    let obj = c.typeImpls.getOrDefault(lit.strings[types[ch].litId])
-    if viaPointer:
-      c.forwardedDecls.add obj, decl
-    else:
-      if not containsOrIncl(c.lookedAt, obj.int):
-        traverseObject(types, lit, c, obj)
-      c.ordered.add obj, decl
-  of ArrayTy:
-    if viaPointer:
-      c.forwardedDecls.add ch, TypedefStruct
-    else:
-      if not containsOrIncl(c.lookedAt, ch.int):
-        traverseObject(types, lit, c, ch)
-      c.ordered.add ch, TypedefStruct
-  else:
-    discard "uninteresting type as we only focus on the required struct declarations"
-
-proc traverseObject(types: TypeGraph; lit: Literals; c: var TypeOrder; t: TypeId) =
-  for x in sons(types, t):
-    case types[x].kind
-    of FieldDecl:
-      recordDependency types, lit, c, t, x.firstSon
-    of ObjectTy:
-      # inheritance
-      recordDependency types, lit, c, t, x
-    else: discard
-
-proc traverseTypes(types: TypeGraph; lit: Literals; c: var TypeOrder) =
-  for t in allTypes(types):
-    if types[t].kind in {ObjectDecl, UnionDecl}:
-      assert types[t.firstSon].kind == NameVal
-      c.typeImpls[lit.strings[types[t.firstSon].litId]] = t
-
-  for t in allTypesIncludingInner(types):
-    case types[t].kind
-    of ObjectDecl, UnionDecl:
-      traverseObject types, lit, c, t
-      let decl = if types[t].kind == ObjectDecl: TypedefStruct else: TypedefUnion
-      c.ordered.add t, decl
-    of ArrayTy:
-      traverseObject types, lit, c, t
-      c.ordered.add t, TypedefStruct
-    else: discard
-
-when false:
-  template emitType(s: string) = c.types.add c.tokens.getOrIncl(s)
-  template emitType(t: Token) = c.types.add t
-  template emitType(t: PredefinedToken) = c.types.add Token(t)
-
-proc genType(g: var GeneratedCode; types: TypeGraph; lit: Literals; t: TypeId; name = "") =
-  template maybeAddName =
-    if name != "":
-      g.add Space
-      g.add name
-
-  template atom(s: string) =
-    g.add s
-    maybeAddName()
-  case types[t].kind
-  of VoidTy: atom "void"
-  of IntTy: atom "NI" & $types[t].integralBits
-  of UIntTy: atom "NU" & $types[t].integralBits
-  of FloatTy: atom "NF" & $types[t].integralBits
-  of BoolTy: atom "NB" & $types[t].integralBits
-  of CharTy: atom "NC" & $types[t].integralBits
-  of ObjectTy, UnionTy, NameVal, AnnotationVal:
-    atom lit.strings[types[t].litId]
-  of VarargsTy:
-    g.add "..."
-  of APtrTy, UPtrTy, AArrayPtrTy, UArrayPtrTy:
-    genType g, types, lit, elementType(types, t)
-    g.add Star
-    maybeAddName()
-  of ArrayTy:
-    genType g, types, lit, arrayName(types, t)
-    maybeAddName()
-  of LastArrayTy:
-    genType g, types, lit, elementType(types, t)
-    maybeAddName()
-    g.add BracketLe
-    g.add BracketRi
-  of ProcTy:
-    let (retType, callConv) = returnType(types, t)
-    genType g, types, lit, retType
-    g.add Space
-    g.add ParLe
-    genType g, types, lit, callConv
-    g.add Star # "(*fn)"
-    maybeAddName()
-    g.add ParRi
-    g.add ParLe
-    var i = 0
-    for ch in params(types, t):
-      if i > 0: g.add Comma
-      genType g, types, lit, ch
-      inc i
-    g.add ParRi
-  of ObjectDecl, UnionDecl:
-    atom lit.strings[types[t.firstSon].litId]
-  of IntVal, SizeVal, AlignVal, OffsetVal, FieldDecl:
-    #raiseAssert "did not expect: " & $types[t].kind
-    g.add "BUG "
-    atom $types[t].kind
-
-proc generateTypes(g: var GeneratedCode; types: TypeGraph; lit: Literals; c: TypeOrder) =
-  for (t, declKeyword) in c.forwardedDecls.s:
-    let name = if types[t].kind == ArrayTy: arrayName(types, t) else: t.firstSon
-    let s {.cursor.} = lit.strings[types[name].litId]
-    g.add declKeyword
-    g.add s
-    g.add Space
-    g.add s
-    g.add Semicolon
-
-  for (t, declKeyword) in c.ordered.s:
-    let name = if types[t].kind == ArrayTy: arrayName(types, t) else: t.firstSon
-    let litId = types[name].litId
-    if not g.generatedTypes.containsOrIncl(litId.int):
-      let s {.cursor.} = lit.strings[litId]
-      g.add declKeyword
-      g.add CurlyLe
-      if types[t].kind == ArrayTy:
-        genType g, types, lit, elementType(types, t), "a"
-        g.add BracketLe
-        g.add $arrayLen(types, t)
-        g.add BracketRi
-        g.add Semicolon
-      else:
-        var i = 0
-        for x in sons(types, t):
-          case types[x].kind
-          of FieldDecl:
-            genType g, types, lit, x.firstSon, "F" & $i
-            g.add Semicolon
-            inc i
-          of ObjectTy:
-            genType g, types, lit, x, "P"
-            g.add Semicolon
-          else: discard
-      g.add CurlyRi
-      g.add s
-      g.add Semicolon
-
-# Procs
-
-proc toCChar*(c: char; result: var string) {.inline.} =
-  case c
-  of '\0'..'\x1F', '\x7F'..'\xFF':
-    result.add '\\'
-    result.add toOctal(c)
-  of '\'', '\"', '\\', '?':
-    result.add '\\'
-    result.add c
-  else:
-    result.add c
-
-proc makeCString(s: string): string =
-  result = newStringOfCap(s.len + 10)
-  result.add('"')
-  for c in s: toCChar(c, result)
-  result.add('"')
-
-template emitData(s: string) = c.data.add c.tokens.getOrIncl(s)
-template emitData(t: Token) = c.data.add t
-template emitData(t: PredefinedToken) = c.data.add Token(t)
-
-proc genStrLit(c: var GeneratedCode; lit: Literals; litId: LitId): Token =
-  result = Token(c.tokens.getOrIncl "QStr" & $litId)
-  if not containsOrIncl(c.emittedStrings, int(litId)):
-    let s {.cursor.} = lit.strings[litId]
-    emitData "static const struct "
-    emitData CurlyLe
-    emitData "NI cap"
-    emitData Semicolon
-    emitData "NC8 data"
-    emitData BracketLe
-    emitData $s.len
-    emitData "+1"
-    emitData BracketRi
-    emitData Semicolon
-    emitData CurlyRi
-    emitData result
-    emitData AsgnOpr
-    emitData CurlyLe
-    emitData $s.len
-    emitData " | NIM_STRLIT_FLAG"
-    emitData Comma
-    emitData makeCString(s)
-    emitData CurlyRi
-    emitData Semicolon
-
-proc genIntLit(c: var GeneratedCode; lit: Literals; litId: LitId) =
-  let i = lit.numbers[litId]
-  if i > low(int32) and i <= high(int32):
-    c.add $i
-  elif i == low(int32):
-    # Nim has the same bug for the same reasons :-)
-    c.add "(-2147483647 -1)"
-  elif i > low(int64):
-    c.add "IL64("
-    c.add $i
-    c.add ")"
-  else:
-    c.add "(IL64(-9223372036854775807) - IL64(1))"
-
-proc gen(c: var GeneratedCode; t: Tree; n: NodePos)
-
-proc genDisplayName(c: var GeneratedCode; symId: SymId) =
-  let displayName = c.m.symnames[symId]
-  if displayName != LitId(0):
-    c.add "/*"
-    c.add c.m.lit.strings[displayName]
-    c.add "*/"
-
-proc genSymDef(c: var GeneratedCode; t: Tree; n: NodePos) =
-  if t[n].kind == SymDef:
-    let symId = t[n].symId
-    c.needsPrefix.incl symId.int
-    genDisplayName c, symId
-  gen c, t, n
-
-proc genGlobal(c: var GeneratedCode; t: Tree; name, typ: NodePos; annotation: string) =
-  c.add annotation
-  let m: string
-  if t[name].kind == SymDef:
-    let symId = t[name].symId
-    m = c.tokens[mangleModuleName(c, c.m.namespace)] & "__" & $symId
-    genDisplayName c, symId
-  else:
-    assert t[name].kind == ModuleSymUse
-    let (x, y) = sons2(t, name)
-    m = c.tokens[mangleModuleName(c, t[x].litId)] & "__" & $t[y].immediateVal
-  genType c, c.m.types, c.m.lit, t[typ].typeId, m
-
-proc genLocal(c: var GeneratedCode; t: Tree; name, typ: NodePos; annotation: string) =
-  assert t[name].kind == SymDef
-  c.add annotation
-  let symId = t[name].symId
-  genType c, c.m.types, c.m.lit, t[typ].typeId, "q" & $symId
-  genDisplayName c, symId
-
-proc genProcDecl(c: var GeneratedCode; t: Tree; n: NodePos; isExtern: bool) =
-  let signatureBegin = c.code.len
-  let name = n.firstSon
-
-  var prc = n.firstSon
-  next t, prc
-
-  while true:
-    case t[prc].kind
-    of PragmaPair:
-      let (x, y) = sons2(t, prc)
-      let key = cast[PragmaKey](t[x].rawOperand)
-      case key
-      of HeaderImport:
-        let lit = t[y].litId
-        let headerAsStr {.cursor.} = c.m.lit.strings[lit]
-        let header = c.tokens.getOrIncl(headerAsStr)
-        # headerAsStr can be empty, this has the semantics of the `nodecl` pragma:
-        if headerAsStr.len > 0 and not c.includedHeaders.containsOrIncl(int header):
-          if headerAsStr[0] == '#':
-            discard "skip the #include"
-          else:
-            c.includes.add Token(IncludeKeyword)
-          c.includes.add header
-          c.includes.add Token NewLine
-        # do not generate code for importc'ed procs:
-        return
-      of DllImport:
-        let lit = t[y].litId
-        raiseAssert "cannot eval: " & c.m.lit.strings[lit]
-      else: discard
-    of PragmaId: discard
-    else: break
-    next t, prc
-
-  var resultDeclPos = NodePos(-1)
-  if t[prc].kind == SummonResult:
-    resultDeclPos = prc
-    gen c, t, prc.firstSon
-    next t, prc
-  else:
-    c.add "void"
-  c.add Space
-  genSymDef c, t, name
-  c.add ParLe
-  var params = 0
-  while t[prc].kind == SummonParam:
-    if params > 0: c.add Comma
-    let (typ, sym) = sons2(t, prc)
-    genLocal c, t, sym, typ, ""
-    next t, prc
-    inc params
-  if params == 0:
-    c.add "void"
-  c.add ParRi
-
-  for i in signatureBegin ..< c.code.len:
-    c.protos.add c.code[i]
-  c.protos.add Token Semicolon
-
-  if isExtern:
-    c.code.setLen signatureBegin
-  else:
-    c.add CurlyLe
-    if resultDeclPos.int >= 0:
-      gen c, t, resultDeclPos
-    for ch in sonsRest(t, n, prc):
-      gen c, t, ch
-    c.add CurlyRi
-
-template triop(opr) =
-  let (typ, a, b) = sons3(t, n)
-  c.add ParLe
-  c.add ParLe
-  gen c, t, typ
-  c.add ParRi
-  gen c, t, a
-  c.add opr
-  gen c, t, b
-  c.add ParRi
-
-template cmpop(opr) =
-  let (_, a, b) = sons3(t, n)
-  c.add ParLe
-  gen c, t, a
-  c.add opr
-  gen c, t, b
-  c.add ParRi
-
-template binaryop(opr) =
-  let (typ, a) = sons2(t, n)
-  c.add ParLe
-  c.add ParLe
-  gen c, t, typ
-  c.add ParRi
-  c.add opr
-  gen c, t, a
-  c.add ParRi
-
-template checkedBinaryop(opr) =
-  let (typ, labIdx, a, b) = sons4(t, n)
-  let bits = integralBits(c.m.types[t[typ].typeId])
-  let lab = t[labIdx].label
-
-  c.add (opr & $bits)
-  c.add ParLe
-  c.gen t, a
-  c.add Comma
-  c.gen t, b
-  c.add Comma
-  c.add "L" & $lab.int
-  c.add ParRi
-
-proc genNumberConv(c: var GeneratedCode; t: Tree; n: NodePos) =
-  let (typ, arg) = sons2(t, n)
-  if t[arg].kind == IntVal:
-    let litId = t[arg].litId
-    c.add ParLe
-    c.add ParLe
-    gen c, t, typ
-    c.add ParRi
-    case c.m.types[t[typ].typeId].kind
-    of UIntTy:
-      let x = cast[uint64](c.m.lit.numbers[litId])
-      c.add $x
-    of FloatTy:
-      let x = cast[float64](c.m.lit.numbers[litId])
-      c.add $x
-    else:
-      gen c, t, arg
-    c.add ParRi
-  else:
-    binaryop ""
-
-template moveToDataSection(body: untyped) =
-  let oldLen = c.code.len
-  body
-  for i in oldLen ..< c.code.len:
-    c.data.add c.code[i]
-  setLen c.code, oldLen
-
-proc gen(c: var GeneratedCode; t: Tree; n: NodePos) =
-  case t[n].kind
-  of Nop:
-    discard "nothing to emit"
-  of ImmediateVal:
-    c.add $t[n].immediateVal
-  of IntVal:
-    genIntLit c, c.m.lit, t[n].litId
-  of StrVal:
-    c.code.add genStrLit(c, c.m.lit, t[n].litId)
-  of Typed:
-    genType c, c.m.types, c.m.lit, t[n].typeId
-  of SymDef, SymUse:
-    let s = t[n].symId
-    if c.needsPrefix.contains(s.int):
-      c.code.add mangleModuleName(c, c.m.namespace)
-      c.add "__"
-      c.add $s
-    else:
-      # XXX Use proper names here
-      c.add "q"
-      c.add $s
-
-  of ModuleSymUse:
-    let (x, y) = sons2(t, n)
-    let u = mangleModuleName(c, t[x].litId)
-    let s = t[y].immediateVal
-    c.code.add u
-    c.add "__"
-    c.add $s
-
-  of NilVal:
-    c.add NullPtr
-  of LoopLabel:
-    c.add WhileKeyword
-    c.add ParLe
-    c.add "1"
-    c.add ParRi
-    c.add CurlyLe
-  of GotoLoop:
-    c.add CurlyRi
-  of Label:
-    let lab = t[n].label
-    c.add "L"
-    c.add $lab.int
-    c.add Colon
-    c.add Semicolon
-  of Goto:
-    let lab = t[n].label
-    c.add "goto L"
-    c.add $lab.int
-    c.add Semicolon
-  of CheckedGoto:
-    discard "XXX todo"
-  of ArrayConstr:
-    c.add CurlyLe
-    c.add ".a = "
-    c.add CurlyLe
-    var i = 0
-    for ch in sonsFrom1(t, n):
-      if i > 0: c.add Comma
-      c.gen t, ch
-      inc i
-    c.add CurlyRi
-    c.add CurlyRi
-  of ObjConstr:
-    c.add CurlyLe
-    var i = 0
-    for ch in sonsFrom1(t, n):
-      if i mod 2 == 0:
-        if i > 0: c.add Comma
-        c.add ".F" & $t[ch].immediateVal
-        c.add AsgnOpr
-      else:
-        c.gen t, ch
-      inc i
-    c.add CurlyRi
-  of Ret:
-    c.add ReturnKeyword
-    c.gen t, n.firstSon
-    c.add Semicolon
-  of Select:
-    c.add SwitchKeyword
-    c.add ParLe
-    let (_, selector) = sons2(t, n)
-    c.gen t, selector
-    c.add ParRi
-    c.add CurlyLe
-    for ch in sonsFromN(t, n, 2):
-      c.gen t, ch
-    c.add CurlyRi
-  of SelectPair:
-    let (le, ri) = sons2(t, n)
-    c.gen t, le
-    c.gen t, ri
-  of SelectList:
-    for ch in sons(t, n):
-      c.gen t, ch
-  of SelectValue:
-    c.add CaseKeyword
-    c.gen t, n.firstSon
-    c.add Colon
-  of SelectRange:
-    let (le, ri) = sons2(t, n)
-    c.add CaseKeyword
-    c.gen t, le
-    c.add " ... "
-    c.gen t, ri
-    c.add Colon
-  of ForeignDecl:
-    c.data.add LitId(ExternKeyword)
-    c.gen t, n.firstSon
-  of SummonGlobal:
-    moveToDataSection:
-      let (typ, sym) = sons2(t, n)
-      c.genGlobal t, sym, typ, ""
-      c.add Semicolon
-  of SummonThreadLocal:
-    moveToDataSection:
-      let (typ, sym) = sons2(t, n)
-      c.genGlobal t, sym, typ, "__thread "
-      c.add Semicolon
-  of SummonConst:
-    moveToDataSection:
-      let (typ, sym, val) = sons3(t, n)
-      c.genGlobal t, sym, typ, "const "
-      c.add AsgnOpr
-      c.gen t, val
-      c.add Semicolon
-  of Summon, SummonResult:
-    let (typ, sym) = sons2(t, n)
-    c.genLocal t, sym, typ, ""
-    c.add Semicolon
-
-  of SummonParam:
-    raiseAssert "SummonParam should have been handled in genProc"
-  of Kill:
-    discard "we don't care about Kill instructions"
-  of AddrOf:
-    let (_, arg) = sons2(t, n)
-    c.add "&"
-    gen c, t, arg
-  of DerefArrayAt:
-    let (_, a, i) = sons3(t, n)
-    gen c, t, a
-    c.add BracketLe
-    gen c, t, i
-    c.add BracketRi
-  of ArrayAt:
-    let (_, a, i) = sons3(t, n)
-    gen c, t, a
-    c.add Dot
-    c.add "a"
-    c.add BracketLe
-    gen c, t, i
-    c.add BracketRi
-  of FieldAt:
-    let (_, a, b) = sons3(t, n)
-    gen c, t, a
-    let field = t[b].immediateVal
-    c.add Dot
-    c.add "F" & $field
-  of DerefFieldAt:
-    let (_, a, b) = sons3(t, n)
-    gen c, t, a
-    let field = t[b].immediateVal
-    c.add Arrow
-    c.add "F" & $field
-  of Load:
-    let (_, arg) = sons2(t, n)
-    c.add ParLe
-    c.add "*"
-    gen c, t, arg
-    c.add ParRi
-  of Store:
-    raiseAssert "Assumption was that Store is unused!"
-  of Asgn:
-    let (_, dest, src) = sons3(t, n)
-    gen c, t, dest
-    c.add AsgnOpr
-    gen c, t, src
-    c.add Semicolon
-  of CheckedRange:
-    c.add "nimCheckRange"
-    c.add ParLe
-    let (_, gotoInstr, x, a, b) = sons5(t, n)
-    gen c, t, x
-    c.add Comma
-    gen c, t, a
-    c.add Comma
-    gen c, t, b
-    c.add Comma
-    c.add "L" & $t[gotoInstr].label.int
-    c.add ParRi
-  of CheckedIndex:
-    c.add "nimCheckIndex"
-    c.add ParLe
-    let (gotoInstr, x, a) = sons3(t, n)
-    gen c, t, x
-    c.add Comma
-    gen c, t, a
-    c.add Comma
-    c.add "L" & $t[gotoInstr].label.int
-    c.add ParRi
-  of Call, IndirectCall:
-    let (typ, fn) = sons2(t, n)
-    gen c, t, fn
-    c.add ParLe
-    var i = 0
-    for ch in sonsFromN(t, n, 2):
-      if i > 0: c.add Comma
-      gen c, t, ch
-      inc i
-    c.add ParRi
-    if c.m.types[t[typ].typeId].kind == VoidTy:
-      c.add Semicolon
-  of CheckedCall, CheckedIndirectCall:
-    let (typ, gotoInstr, fn) = sons3(t, n)
-    gen c, t, fn
-    c.add ParLe
-    var i = 0
-    for ch in sonsFromN(t, n, 3):
-      if i > 0: c.add Comma
-      gen c, t, ch
-      inc i
-    c.add ParRi
-    if c.m.types[t[typ].typeId].kind == VoidTy:
-      c.add Semicolon
-
-  of CheckedAdd: checkedBinaryop "nimAddInt"
-  of CheckedSub: checkedBinaryop "nimSubInt"
-  of CheckedMul: checkedBinaryop "nimMulInt"
-  of CheckedDiv: checkedBinaryop "nimDivInt"
-  of CheckedMod: checkedBinaryop "nimModInt"
-  of Add: triop " + "
-  of Sub: triop " - "
-  of Mul: triop " * "
-  of Div: triop " / "
-  of Mod: triop " % "
-  of BitShl: triop " << "
-  of BitShr: triop " >> "
-  of BitAnd: triop " & "
-  of BitOr: triop " | "
-  of BitXor: triop " ^ "
-  of BitNot: binaryop " ~ "
-  of BoolNot: binaryop " !"
-  of Eq: cmpop " == "
-  of Le: cmpop " <= "
-  of Lt: cmpop " < "
-  of Cast: binaryop ""
-  of NumberConv: genNumberConv c, t, n
-  of CheckedObjConv: binaryop ""
-  of ObjConv: binaryop ""
-  of Emit: raiseAssert "cannot interpret: Emit"
-  of ProcDecl: genProcDecl c, t, n, false
-  of ForeignProcDecl: genProcDecl c, t, n, true
-  of PragmaPair, PragmaId, TestOf, Yld, SetExc, TestExc:
-    c.add "cannot interpret: " & $t[n].kind
-
-const
-  Prelude = """
-/* GENERATED CODE. DO NOT EDIT. */
-
-#ifdef __cplusplus
-#define NB8 bool
-#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901)
-// see #13798: to avoid conflicts for code emitting `#include <stdbool.h>`
-#define NB8 _Bool
-#else
-typedef unsigned char NB8; // best effort
-#endif
-
-typedef unsigned char NC8;
-
-typedef float NF32;
-typedef double NF64;
-#if defined(__BORLANDC__) || defined(_MSC_VER)
-typedef signed char NI8;
-typedef signed short int NI16;
-typedef signed int NI32;
-typedef __int64 NI64;
-/* XXX: Float128? */
-typedef unsigned char NU8;
-typedef unsigned short int NU16;
-typedef unsigned int NU32;
-typedef unsigned __int64 NU64;
-#elif defined(HAVE_STDINT_H)
-#ifndef USE_NIM_NAMESPACE
-#  include <stdint.h>
-#endif
-typedef int8_t NI8;
-typedef int16_t NI16;
-typedef int32_t NI32;
-typedef int64_t NI64;
-typedef uint8_t NU8;
-typedef uint16_t NU16;
-typedef uint32_t NU32;
-typedef uint64_t NU64;
-#elif defined(HAVE_CSTDINT)
-#ifndef USE_NIM_NAMESPACE
-#  include <cstdint>
-#endif
-typedef std::int8_t NI8;
-typedef std::int16_t NI16;
-typedef std::int32_t NI32;
-typedef std::int64_t NI64;
-typedef std::uint8_t NU8;
-typedef std::uint16_t NU16;
-typedef std::uint32_t NU32;
-typedef std::uint64_t NU64;
-#else
-/* Unknown compiler/version, do our best */
-#ifdef __INT8_TYPE__
-typedef __INT8_TYPE__ NI8;
-#else
-typedef signed char NI8;
-#endif
-#ifdef __INT16_TYPE__
-typedef __INT16_TYPE__ NI16;
-#else
-typedef signed short int NI16;
-#endif
-#ifdef __INT32_TYPE__
-typedef __INT32_TYPE__ NI32;
-#else
-typedef signed int NI32;
-#endif
-#ifdef __INT64_TYPE__
-typedef __INT64_TYPE__ NI64;
-#else
-typedef long long int NI64;
-#endif
-/* XXX: Float128? */
-#ifdef __UINT8_TYPE__
-typedef __UINT8_TYPE__ NU8;
-#else
-typedef unsigned char NU8;
-#endif
-#ifdef __UINT16_TYPE__
-typedef __UINT16_TYPE__ NU16;
-#else
-typedef unsigned short int NU16;
-#endif
-#ifdef __UINT32_TYPE__
-typedef __UINT32_TYPE__ NU32;
-#else
-typedef unsigned int NU32;
-#endif
-#ifdef __UINT64_TYPE__
-typedef __UINT64_TYPE__ NU64;
-#else
-typedef unsigned long long int NU64;
-#endif
-#endif
-
-#ifdef NIM_INTBITS
-#  if NIM_INTBITS == 64
-typedef NI64 NI;
-typedef NU64 NU;
-#  elif NIM_INTBITS == 32
-typedef NI32 NI;
-typedef NU32 NU;
-#  elif NIM_INTBITS == 16
-typedef NI16 NI;
-typedef NU16 NU;
-#  elif NIM_INTBITS == 8
-typedef NI8 NI;
-typedef NU8 NU;
-#  else
-#    error "invalid bit width for int"
-#  endif
-#endif
-
-#define NIM_STRLIT_FLAG ((NU)(1) << ((NIM_INTBITS) - 2)) /* This has to be the same as system.strlitFlag! */
-
-#define nimAddInt64(a, b, L) ({long long int res; if(__builtin_saddll_overflow(a, b, &res)) goto L; res})
-#define nimSubInt64(a, b, L) ({long long int res; if(__builtin_ssubll_overflow(a, b, &res)) goto L; res})
-#define nimMulInt64(a, b, L) ({long long int res; if(__builtin_smulll_overflow(a, b, &res)) goto L; res})
-
-#define nimAddInt32(a, b, L) ({long int res; if(__builtin_sadd_overflow(a, b, &res)) goto L; res})
-#define nimSubInt32(a, b, L) ({long int res; if(__builtin_ssub_overflow(a, b, &res)) goto L; res})
-#define nimMulInt32(a, b, L) ({long int res; if(__builtin_smul_overflow(a, b, &res)) goto L; res})
-
-#define nimCheckRange(x, a, b, L) ({if (x < a || x > b) goto L; x})
-#define nimCheckIndex(x, a, L) ({if (x >= a) goto L; x})
-
-"""
-
-proc traverseCode(c: var GeneratedCode) =
-  const AllowedInToplevelC = {SummonConst, SummonGlobal, SummonThreadLocal,
-                              ProcDecl, ForeignDecl, ForeignProcDecl}
-  var i = NodePos(0)
-  while i.int < c.m.code.len:
-    let oldLen = c.code.len
-    let moveToInitSection = c.m.code[NodePos(i)].kind notin AllowedInToplevelC
-
-    gen c, c.m.code, NodePos(i)
-    next c.m.code, i
-
-    if moveToInitSection:
-      for i in oldLen ..< c.code.len:
-        c.init.add c.code[i]
-      setLen c.code, oldLen
-
-proc generateCode*(inp, outp: string) =
-  var c = initGeneratedCode(load(inp))
-
-  var co = TypeOrder()
-  traverseTypes(c.m.types, c.m.lit, co)
-
-  generateTypes(c, c.m.types, c.m.lit, co)
-  let typeDecls = move c.code
-
-  traverseCode c
-  var f = CppFile(f: open(outp, fmWrite))
-  f.write "#define NIM_INTBITS " & $c.m.intbits & "\n"
-  f.write Prelude
-  writeTokenSeq f, c.includes, c
-  writeTokenSeq f, typeDecls, c
-  writeTokenSeq f, c.data, c
-  writeTokenSeq f, c.protos, c
-  writeTokenSeq f, c.code, c
-  if c.init.len > 0:
-    f.write "void __attribute__((constructor)) init(void) {"
-    writeTokenSeq f, c.init, c
-    f.write "}\n\n"
-  f.f.close

compiler/nir/nir.nim

@@ -1,105 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Nim Intermediate Representation, designed to capture all of Nim's semantics without losing too much
-## precious information. Can easily be translated into C. And to JavaScript, hopefully.
-
-from std/os import addFileExt, `/`, createDir
-
-import std / assertions
-import ".." / [ast, modulegraphs, renderer, transf, options, msgs, lineinfos]
-import nirtypes, nirinsts, ast2ir, nirlineinfos, nirfiles, nirvm
-
-import ".." / ic / [rodfiles, bitabs]
-
-type
-  PCtx* = ref object of TPassContext
-    m: ModuleCon
-    c: ProcCon
-    oldErrorCount: int
-    bytecode: Bytecode
-
-proc newCtx*(module: PSym; g: ModuleGraph; idgen: IdGenerator): PCtx =
-  var lit = Literals()
-  var nirm = (ref NirModule)(types: initTypeGraph(lit), lit: lit)
-  var m = initModuleCon(g, g.config, idgen, module, nirm)
-  m.noModularity = true
-  PCtx(m: m, c: initProcCon(m, nil, g.config), idgen: idgen, bytecode: initBytecode(nirm))
-
-proc refresh*(c: PCtx; module: PSym; idgen: IdGenerator) =
-  #c.m = initModuleCon(c.m.graph, c.m.graph.config, idgen, module, c.m.nirm)
-  #c.m.noModularity = true
-  c.c = initProcCon(c.m, nil, c.m.graph.config)
-  c.idgen = idgen
-
-proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) =
-  if graph.repl.isNil:
-    graph.repl = newCtx(module, graph, idgen)
-    #registerAdditionalOps(PCtx graph.repl)
-  else:
-    refresh(PCtx graph.repl, module, idgen)
-
-proc setupNirReplGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
-  setupGlobalCtx(module, graph, idgen)
-  result = PCtx graph.repl
-
-proc evalStmt(c: PCtx; n: PNode) =
-  let n = transformExpr(c.m.graph, c.idgen, c.m.module, n)
-  let pc = genStmt(c.c, n)
-  #var res = ""
-  #toString c.m.nirm.code, NodePos(pc), c.m.nirm.lit.strings, c.m.nirm.lit.numbers, c.m.symnames, res
-  #res.add "\n--------------------------\n"
-  #toString res, c.m.types.g
-  if pc.int < c.m.nirm.code.len:
-    c.bytecode.interactive = c.m.graph.interactive
-    execCode c.bytecode, c.m.nirm.code, pc
-  #echo res
-
-proc runCode*(c: PPassContext; n: PNode): PNode =
-  let c = PCtx(c)
-  # don't eval errornous code:
-  if c.oldErrorCount == c.m.graph.config.errorCounter:
-    evalStmt(c, n)
-    result = newNodeI(nkEmpty, n.info)
-  else:
-    result = n
-  c.oldErrorCount = c.m.graph.config.errorCounter
-
-type
-  NirPassContext* = ref object of TPassContext
-    m: ModuleCon
-    c: ProcCon
-
-proc openNirBackend*(g: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
-  var lit = Literals()
-  var nirm = (ref NirModule)(types: initTypeGraph(lit), lit: lit)
-  let m = initModuleCon(g, g.config, idgen, module, nirm)
-  NirPassContext(m: m, c: initProcCon(m, nil, g.config), idgen: idgen)
-
-proc gen(c: NirPassContext; n: PNode) =
-  let n = transformExpr(c.m.graph, c.idgen, c.m.module, n)
-  let pc = genStmt(c.c, n)
-
-proc nirBackend*(c: PPassContext; n: PNode): PNode =
-  gen(NirPassContext(c), n)
-  result = n
-
-proc closeNirBackend*(c: PPassContext; finalNode: PNode) =
-  discard nirBackend(c, finalNode)
-
-  let c = NirPassContext(c)
-  let nimcache = getNimcacheDir(c.c.config).string
-  createDir nimcache
-  let outp = nimcache / c.m.module.name.s.addFileExt("nir")
-  #c.m.nirm.code = move c.c.code
-  try:
-    store c.m.nirm[], outp
-    echo "created: ", outp
-  except IOError:
-    rawMessage(c.c.config, errFatal, "serialization failed: " & outp)

compiler/nir/nirc.nim

@@ -1,52 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Nir Compiler.
-
-import ".." / ic / [bitabs, rodfiles]
-import nirinsts, nirtypes, nirlineinfos, nirfiles, cir
-
-proc view(filename: string) =
-  let m = load(filename)
-  var res = ""
-  allTreesToString m.code, m.lit.strings, m.lit.numbers, m.symnames, res
-  res.add "\n# TYPES\n"
-  nirtypes.toString res, m.types
-  echo res
-
-import std / [syncio, parseopt]
-
-proc writeHelp =
-  echo """Usage: nirc view|c <file.nir>"""
-  quit 0
-
-proc main =
-  var inp = ""
-  var cmd = ""
-  for kind, key, val in getopt():
-    case kind
-    of cmdArgument:
-      if cmd.len == 0: cmd = key
-      elif inp.len == 0: inp = key
-      else: quit "Error: too many arguments"
-    of cmdLongOption, cmdShortOption:
-      case key
-      of "help", "h": writeHelp()
-      of "version", "v": stdout.write "1.0\n"
-    of cmdEnd: discard
-  if inp.len == 0:
-    quit "Error: no input file specified"
-  case cmd
-  of "", "view":
-    view inp
-  of "c":
-    let outp = inp & ".c"
-    cir.generateCode inp, outp
-
-main()

compiler/nir/nirfiles.nim

@@ -1,83 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import ".." / ic / [bitabs, rodfiles]
-import nirinsts, nirtypes, nirlineinfos
-
-type
-  NirModule* = object
-    code*: Tree
-    man*: LineInfoManager
-    types*: TypeGraph
-    lit*: Literals
-    namespace*: LitId
-    intbits*: uint32
-    symnames*: SymNames
-
-proc load*(filename: string): NirModule =
-  let lit = Literals()
-  result = NirModule(lit: lit, types: initTypeGraph(lit))
-  var r = rodfiles.open(filename)
-  try:
-    r.loadHeader(nirCookie)
-    r.loadSection stringsSection
-    r.load result.lit.strings
-
-    r.loadSection numbersSection
-    r.load result.lit.numbers
-
-    r.loadSection bodiesSection
-    r.load result.code
-
-    r.loadSection typesSection
-    r.load result.types
-
-    r.loadSection sideChannelSection
-    r.load result.man
-
-    r.loadSection namespaceSection
-    r.loadPrim result.namespace
-    r.loadPrim result.intbits
-
-    r.loadSection symnamesSection
-    r.load result.symnames
-
-  finally:
-    r.close()
-
-proc store*(m: NirModule; outp: string) =
-  var r = rodfiles.create(outp)
-  try:
-    r.storeHeader(nirCookie)
-    r.storeSection stringsSection
-    r.store m.lit.strings
-
-    r.storeSection numbersSection
-    r.store m.lit.numbers
-
-    r.storeSection bodiesSection
-    r.store m.code
-
-    r.storeSection typesSection
-    r.store m.types
-
-    r.storeSection sideChannelSection
-    r.store m.man
-
-    r.storeSection namespaceSection
-    r.storePrim m.namespace
-    r.storePrim m.intbits
-
-    r.storeSection symnamesSection
-    r.store m.symnames
-
-  finally:
-    r.close()
-  if r.err != ok:
-    raise newException(IOError, "could store into: " & outp)

compiler/nir/nirinsts.nim

@@ -1,582 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## NIR instructions. Somewhat inspired by LLVM's instructions.
-
-import std / [assertions, hashes]
-import .. / ic / [bitabs, rodfiles]
-import nirlineinfos, nirtypes
-
-const
-  NirVersion = 1
-  nirCookie* = [byte(0), byte('N'), byte('I'), byte('R'),
-            byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(NirVersion)]
-
-type
-  SymId* = distinct int
-
-proc `$`*(s: SymId): string {.borrow.}
-proc hash*(s: SymId): Hash {.borrow.}
-proc `==`*(a, b: SymId): bool {.borrow.}
-
-type
-  Opcode* = enum
-    Nop,
-    ImmediateVal,
-    IntVal,
-    StrVal,
-    SymDef,
-    SymUse,
-    Typed,   # with type ID
-    PragmaId, # with Pragma ID, possible values: see PragmaKey enum
-    NilVal,
-    Label,
-    Goto,
-    CheckedGoto,
-    LoopLabel,
-    GotoLoop,  # last atom
-
-    ModuleSymUse, # `"module".x`
-
-    ArrayConstr,
-    ObjConstr,
-    Ret,
-    Yld,
-
-    Select,
-    SelectPair,  # ((values...), Label)
-    SelectList,  # (values...)
-    SelectValue, # (value)
-    SelectRange, # (valueA..valueB)
-    ForeignDecl, # Can wrap SummonGlobal, SummonThreadLocal, SummonConst
-    SummonGlobal,
-    SummonThreadLocal,
-    Summon, # x = Summon Typed <Type ID>; x begins to live
-    SummonResult,
-    SummonParam,
-    SummonConst,
-    Kill, # `Kill x`: scope end for `x`
-
-    AddrOf,
-    ArrayAt, # a[i]
-    DerefArrayAt, # a[i] where `a` is a PtrArray; `a[][i]`
-    FieldAt, # obj.field
-    DerefFieldAt, # obj[].field
-
-    Load, # a[]
-    Store, # a[] = b
-    Asgn,  # a = b
-    SetExc,
-    TestExc,
-
-    CheckedRange,
-    CheckedIndex,
-
-    Call,
-    IndirectCall,
-    CheckedCall, # call that can raise
-    CheckedIndirectCall, # call that can raise
-    CheckedAdd, # with overflow checking etc.
-    CheckedSub,
-    CheckedMul,
-    CheckedDiv,
-    CheckedMod,
-    Add,
-    Sub,
-    Mul,
-    Div,
-    Mod,
-    BitShl,
-    BitShr,
-    BitAnd,
-    BitOr,
-    BitXor,
-    BitNot,
-    BoolNot,
-    Eq,
-    Le,
-    Lt,
-    Cast,
-    NumberConv,
-    CheckedObjConv,
-    ObjConv,
-    TestOf,
-    Emit,
-    ProcDecl,
-    ForeignProcDecl,
-    PragmaPair
-
-type
-  PragmaKey* = enum
-    FastCall, StdCall, CDeclCall, SafeCall, SysCall, InlineCall, NoinlineCall, ThisCall, NoCall,
-    CoreName,
-    ExternName,
-    HeaderImport,
-    DllImport,
-    DllExport,
-    ObjExport
-
-const
-  LastAtomicValue = GotoLoop
-
-  OpcodeBits = 8'u32
-  OpcodeMask = (1'u32 shl OpcodeBits) - 1'u32
-
-  ValueProducingAtoms = {ImmediateVal, IntVal, StrVal, SymUse, NilVal}
-
-  ValueProducing* = {
-    ImmediateVal,
-    IntVal,
-    StrVal,
-    SymUse,
-    NilVal,
-    ModuleSymUse,
-    ArrayConstr,
-    ObjConstr,
-    CheckedAdd,
-    CheckedSub,
-    CheckedMul,
-    CheckedDiv,
-    CheckedMod,
-    Add,
-    Sub,
-    Mul,
-    Div,
-    Mod,
-    BitShl,
-    BitShr,
-    BitAnd,
-    BitOr,
-    BitXor,
-    BitNot,
-    BoolNot,
-    Eq,
-    Le,
-    Lt,
-    Cast,
-    NumberConv,
-    CheckedObjConv,
-    ObjConv,
-    AddrOf,
-    Load,
-    ArrayAt,
-    DerefArrayAt,
-    FieldAt,
-    DerefFieldAt,
-    TestOf
-  }
-
-type
-  Instr* = object     # 8 bytes
-    x: uint32
-    info*: PackedLineInfo
-
-template kind*(n: Instr): Opcode = Opcode(n.x and OpcodeMask)
-template operand(n: Instr): uint32 = (n.x shr OpcodeBits)
-
-template rawOperand*(n: Instr): uint32 = (n.x shr OpcodeBits)
-
-template toX(k: Opcode; operand: uint32): uint32 =
-  uint32(k) or (operand shl OpcodeBits)
-
-template toX(k: Opcode; operand: LitId): uint32 =
-  uint32(k) or (operand.uint32 shl OpcodeBits)
-
-type
-  Tree* = object
-    nodes: seq[Instr]
-
-  Values* = object
-    numbers: BiTable[int64]
-    strings: BiTable[string]
-
-type
-  PatchPos* = distinct int
-  NodePos* = distinct int
-
-const
-  InvalidPatchPos* = PatchPos(-1)
-
-proc isValid(p: PatchPos): bool {.inline.} = p.int != -1
-
-proc prepare*(tree: var Tree; info: PackedLineInfo; kind: Opcode): PatchPos =
-  result = PatchPos tree.nodes.len
-  tree.nodes.add Instr(x: toX(kind, 1'u32), info: info)
-
-proc isAtom(tree: Tree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= LastAtomicValue
-proc isAtom(tree: Tree; pos: NodePos): bool {.inline.} = tree.nodes[pos.int].kind <= LastAtomicValue
-
-proc patch*(tree: var Tree; pos: PatchPos) =
-  let pos = pos.int
-  let k = tree.nodes[pos].kind
-  assert k > LastAtomicValue
-  let distance = int32(tree.nodes.len - pos)
-  assert distance > 0
-  tree.nodes[pos].x = toX(k, cast[uint32](distance))
-
-template build*(tree: var Tree; info: PackedLineInfo; kind: Opcode; body: untyped) =
-  let pos = prepare(tree, info, kind)
-  body
-  patch(tree, pos)
-
-template buildTyped*(tree: var Tree; info: PackedLineInfo; kind: Opcode; typ: TypeId; body: untyped) =
-  let pos = prepare(tree, info, kind)
-  tree.addTyped info, typ
-  body
-  patch(tree, pos)
-
-proc len*(tree: Tree): int {.inline.} = tree.nodes.len
-
-template rawSpan(n: Instr): int = int(operand(n))
-
-proc nextChild(tree: Tree; pos: var int) {.inline.} =
-  if tree.nodes[pos].kind > LastAtomicValue:
-    assert tree.nodes[pos].operand > 0'u32
-    inc pos, tree.nodes[pos].rawSpan
-  else:
-    inc pos
-
-proc next*(tree: Tree; pos: var NodePos) {.inline.} = nextChild tree, int(pos)
-
-template firstSon*(n: NodePos): NodePos = NodePos(n.int+1)
-
-template skipTyped*(n: NodePos): NodePos = NodePos(n.int+2)
-
-iterator sons*(tree: Tree; n: NodePos): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > LastAtomicValue
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-iterator sonsFrom1*(tree: Tree; n: NodePos): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > LastAtomicValue
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  nextChild tree, pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-iterator sonsFromN*(tree: Tree; n: NodePos; toSkip = 2): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > LastAtomicValue
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  for i in 1..toSkip:
-    nextChild tree, pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-template `[]`*(t: Tree; n: NodePos): Instr = t.nodes[n.int]
-
-iterator sonsRest*(tree: Tree; parent, n: NodePos): NodePos =
-  var pos = n.int
-  assert tree[parent].kind > LastAtomicValue
-  let last = parent.int + tree[parent].rawSpan
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-proc span(tree: Tree; pos: int): int {.inline.} =
-  if tree.nodes[pos].kind <= LastAtomicValue: 1 else: int(tree.nodes[pos].operand)
-
-proc copyTree*(dest: var Tree; src: Tree) =
-  let pos = 0
-  let L = span(src, pos)
-  let d = dest.nodes.len
-  dest.nodes.setLen(d + L)
-  assert L > 0
-  for i in 0..<L:
-    dest.nodes[d+i] = src.nodes[pos+i]
-
-proc sons2*(tree: Tree; n: NodePos): (NodePos, NodePos) {.inline.} =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  result = (NodePos a, NodePos b)
-
-proc sons3*(tree: Tree; n: NodePos): (NodePos, NodePos, NodePos) {.inline.} =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  result = (NodePos a, NodePos b, NodePos c)
-
-proc sons4*(tree: Tree; n: NodePos): (NodePos, NodePos, NodePos, NodePos) {.inline.} =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  let d = c + span(tree, c)
-  result = (NodePos a, NodePos b, NodePos c, NodePos d)
-
-proc sons5*(tree: Tree; n: NodePos): (NodePos, NodePos, NodePos, NodePos, NodePos) {.inline.} =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  let d = c + span(tree, c)
-  let e = d + span(tree, d)
-  result = (NodePos a, NodePos b, NodePos c, NodePos d, NodePos e)
-
-proc typeId*(ins: Instr): TypeId {.inline.} =
-  assert ins.kind == Typed
-  result = TypeId(ins.operand)
-
-proc symId*(ins: Instr): SymId {.inline.} =
-  assert ins.kind in {SymUse, SymDef}
-  result = SymId(ins.operand)
-
-proc immediateVal*(ins: Instr): int {.inline.} =
-  assert ins.kind == ImmediateVal
-  result = cast[int](ins.operand)
-
-proc litId*(ins: Instr): LitId {.inline.} =
-  assert ins.kind in {StrVal, IntVal}
-  result = LitId(ins.operand)
-
-
-type
-  LabelId* = distinct int
-
-proc `==`*(a, b: LabelId): bool {.borrow.}
-proc hash*(a: LabelId): Hash {.borrow.}
-
-proc label*(ins: Instr): LabelId {.inline.} =
-  assert ins.kind in {Label, LoopLabel, Goto, GotoLoop, CheckedGoto}
-  result = LabelId(ins.operand)
-
-proc newLabel*(labelGen: var int): LabelId {.inline.} =
-  result = LabelId labelGen
-  inc labelGen
-
-proc newLabels*(labelGen: var int; n: int): LabelId {.inline.} =
-  result = LabelId labelGen
-  inc labelGen, n
-
-proc addNewLabel*(t: var Tree; labelGen: var int; info: PackedLineInfo; k: Opcode): LabelId =
-  assert k in {Label, LoopLabel}
-  result = LabelId labelGen
-  t.nodes.add Instr(x: toX(k, uint32(result)), info: info)
-  inc labelGen
-
-proc gotoLabel*(t: var Tree; info: PackedLineInfo; k: Opcode; L: LabelId) =
-  assert k in {Goto, GotoLoop, CheckedGoto}
-  t.nodes.add Instr(x: toX(k, uint32(L)), info: info)
-
-proc addLabel*(t: var Tree; info: PackedLineInfo; k: Opcode; L: LabelId) {.inline.} =
-  assert k in {Label, LoopLabel, Goto, GotoLoop, CheckedGoto}
-  t.nodes.add Instr(x: toX(k, uint32(L)), info: info)
-
-proc addSymUse*(t: var Tree; info: PackedLineInfo; s: SymId) {.inline.} =
-  t.nodes.add Instr(x: toX(SymUse, uint32(s)), info: info)
-
-proc addSymDef*(t: var Tree; info: PackedLineInfo; s: SymId) {.inline.} =
-  t.nodes.add Instr(x: toX(SymDef, uint32(s)), info: info)
-
-proc addNop*(t: var Tree; info: PackedLineInfo) {.inline.} =
-  t.nodes.add Instr(x: toX(Nop, 0'u32), info: info)
-
-proc addTyped*(t: var Tree; info: PackedLineInfo; typ: TypeId) {.inline.} =
-  assert typ.int >= 0
-  t.nodes.add Instr(x: toX(Typed, uint32(typ)), info: info)
-
-proc addSummon*(t: var Tree; info: PackedLineInfo; s: SymId; typ: TypeId; opc = Summon) {.inline.} =
-  assert typ.int >= 0
-  assert opc in {Summon, SummonConst, SummonGlobal, SummonThreadLocal, SummonParam, SummonResult}
-  let x = prepare(t, info, opc)
-  t.nodes.add Instr(x: toX(Typed, uint32(typ)), info: info)
-  t.nodes.add Instr(x: toX(SymDef, uint32(s)), info: info)
-  patch t, x
-
-proc addImmediateVal*(t: var Tree; info: PackedLineInfo; x: int) =
-  assert x >= 0 and x < ((1 shl 32) - OpcodeBits.int)
-  t.nodes.add Instr(x: toX(ImmediateVal, uint32(x)), info: info)
-
-proc addPragmaId*(t: var Tree; info: PackedLineInfo; x: PragmaKey) =
-  t.nodes.add Instr(x: toX(PragmaId, uint32(x)), info: info)
-
-proc addIntVal*(t: var Tree; integers: var BiTable[int64]; info: PackedLineInfo; typ: TypeId; x: int64) =
-  buildTyped t, info, NumberConv, typ:
-    t.nodes.add Instr(x: toX(IntVal, uint32(integers.getOrIncl(x))), info: info)
-
-proc boolVal*(t: var Tree; integers: var BiTable[int64]; info: PackedLineInfo; b: bool) =
-  buildTyped t, info, NumberConv, Bool8Id:
-    t.nodes.add Instr(x: toX(IntVal, uint32(integers.getOrIncl(ord b))), info: info)
-
-proc addStrVal*(t: var Tree; strings: var BiTable[string]; info: PackedLineInfo; s: string) =
-  t.nodes.add Instr(x: toX(StrVal, uint32(strings.getOrIncl(s))), info: info)
-
-proc addStrLit*(t: var Tree; info: PackedLineInfo; s: LitId) =
-  t.nodes.add Instr(x: toX(StrVal, uint32(s)), info: info)
-
-proc addNilVal*(t: var Tree; info: PackedLineInfo; typ: TypeId) =
-  buildTyped t, info, NumberConv, typ:
-    t.nodes.add Instr(x: toX(NilVal, uint32(0)), info: info)
-
-proc store*(r: var RodFile; t: Tree) = storeSeq r, t.nodes
-proc load*(r: var RodFile; t: var Tree) = loadSeq r, t.nodes
-
-proc escapeToNimLit*(s: string; result: var string) =
-  result.add '"'
-  for c in items s:
-    if c < ' ' or int(c) >= 128:
-      result.add '\\'
-      result.addInt int(c)
-    elif c == '\\':
-      result.add r"\\"
-    elif c == '\n':
-      result.add r"\n"
-    elif c == '\r':
-      result.add r"\r"
-    elif c == '\t':
-      result.add r"\t"
-    else:
-      result.add c
-  result.add '"'
-
-type
-  SymNames* = object
-    s: seq[LitId]
-
-proc `[]=`*(t: var SymNames; key: SymId; val: LitId) =
-  let k = int(key)
-  if k >= t.s.len: t.s.setLen k+1
-  t.s[k] = val
-
-proc `[]`*(t: SymNames; key: SymId): LitId =
-  let k = int(key)
-  if k < t.s.len: result = t.s[k]
-  else: result = LitId(0)
-
-template localName(s: SymId): string =
-  let name = names[s]
-  if name != LitId(0):
-    strings[name]
-  else:
-    $s.int
-
-proc store*(r: var RodFile; t: SymNames) = storeSeq(r, t.s)
-proc load*(r: var RodFile; t: var SymNames) = loadSeq(r, t.s)
-
-proc toString*(t: Tree; pos: NodePos; strings: BiTable[string]; integers: BiTable[int64];
-               names: SymNames;
-               r: var string; nesting = 0) =
-  if r.len > 0 and r[r.len-1] notin {' ', '\n', '(', '[', '{'}:
-    r.add ' '
-
-  case t[pos].kind
-  of Nop: r.add "Nop"
-  of ImmediateVal:
-    r.add $t[pos].operand
-  of IntVal:
-    r.add "IntVal "
-    r.add $integers[LitId t[pos].operand]
-  of StrVal:
-    escapeToNimLit(strings[LitId t[pos].operand], r)
-  of SymDef:
-    r.add "SymDef "
-    r.add localName(SymId t[pos].operand)
-  of SymUse:
-    r.add "SymUse "
-    r.add localName(SymId t[pos].operand)
-  of PragmaId:
-    r.add $cast[PragmaKey](t[pos].operand)
-  of Typed:
-    r.add "T<"
-    r.add $t[pos].operand
-    r.add ">"
-  of NilVal:
-    r.add "NilVal"
-  of Label:
-    # undo the nesting:
-    var spaces = r.len-1
-    while spaces >= 0 and r[spaces] == ' ': dec spaces
-    r.setLen spaces+1
-    r.add "\n  L"
-    r.add $t[pos].operand
-  of Goto, CheckedGoto, LoopLabel, GotoLoop:
-    r.add $t[pos].kind
-    r.add " L"
-    r.add $t[pos].operand
-  else:
-    r.add $t[pos].kind
-    r.add "{\n"
-    for i in 0..<(nesting+1)*2: r.add ' '
-    for p in sons(t, pos):
-      toString t, p, strings, integers, names, r, nesting+1
-    r.add "\n"
-    for i in 0..<nesting*2: r.add ' '
-    r.add "}"
-
-proc allTreesToString*(t: Tree; strings: BiTable[string]; integers: BiTable[int64];
-                       names: SymNames;
-                       r: var string) =
-  var i = 0
-  while i < t.len:
-    toString t, NodePos(i), strings, integers, names, r
-    nextChild t, i
-
-type
-  Value* = distinct Tree
-
-proc prepare*(dest: var Value; info: PackedLineInfo; k: Opcode): PatchPos {.inline.} =
-  assert k in ValueProducing - ValueProducingAtoms
-  result = prepare(Tree(dest), info, k)
-
-proc patch*(dest: var Value; pos: PatchPos) {.inline.} =
-  patch(Tree(dest), pos)
-
-proc localToValue*(info: PackedLineInfo; s: SymId): Value =
-  result = Value(Tree())
-  Tree(result).addSymUse info, s
-
-proc hasValue*(v: Value): bool {.inline.} = Tree(v).len > 0
-
-proc isEmpty*(v: Value): bool {.inline.} = Tree(v).len == 0
-
-proc extractTemp*(v: Value): SymId =
-  if hasValue(v) and Tree(v)[NodePos 0].kind == SymUse:
-    result = SymId(Tree(v)[NodePos 0].operand)
-  else:
-    result = SymId(-1)
-
-proc copyTree*(dest: var Tree; src: Value) = copyTree dest, Tree(src)
-
-proc addImmediateVal*(t: var Value; info: PackedLineInfo; x: int) =
-  assert x >= 0 and x < ((1 shl 32) - OpcodeBits.int)
-  Tree(t).nodes.add Instr(x: toX(ImmediateVal, uint32(x)), info: info)
-
-template build*(tree: var Value; info: PackedLineInfo; kind: Opcode; body: untyped) =
-  let pos = prepare(Tree(tree), info, kind)
-  body
-  patch(tree, pos)
-
-proc addTyped*(t: var Value; info: PackedLineInfo; typ: TypeId) {.inline.} =
-  addTyped(Tree(t), info, typ)
-
-template buildTyped*(tree: var Value; info: PackedLineInfo; kind: Opcode; typ: TypeId; body: untyped) =
-  let pos = prepare(tree, info, kind)
-  tree.addTyped info, typ
-  body
-  patch(tree, pos)
-
-proc addStrVal*(t: var Value; strings: var BiTable[string]; info: PackedLineInfo; s: string) =
-  addStrVal(Tree(t), strings, info, s)
-
-proc addNilVal*(t: var Value; info: PackedLineInfo; typ: TypeId) =
-  addNilVal Tree(t), info, typ
-
-proc addIntVal*(t: var Value; integers: var BiTable[int64]; info: PackedLineInfo; typ: TypeId; x: int64) =
-  addIntVal Tree(t), integers, info, typ, x

compiler/nir/nirslots.nim

@@ -1,104 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Management of slots. Similar to "register allocation"
-## in lower level languages.
-
-import std / [assertions, tables]
-import nirtypes, nirinsts
-
-type
-  SlotManagerFlag* = enum
-    ReuseTemps,
-    ReuseVars
-  SlotKind* = enum
-    Temp, Perm
-  SlotManager* = object # "register allocator"
-    live: Table[SymId, (SlotKind, TypeId)]
-    dead: Table[TypeId, seq[SymId]]
-    flags: set[SlotManagerFlag]
-    inScope: seq[SymId]
-
-proc initSlotManager*(flags: set[SlotManagerFlag]): SlotManager {.inline.} =
-  SlotManager(flags: flags)
-
-proc allocRaw(m: var SlotManager; t: TypeId; f: SlotManagerFlag; k: SlotKind;
-              symIdgen: var int32): SymId {.inline.} =
-  if f in m.flags and m.dead.hasKey(t) and m.dead[t].len > 0:
-    result = m.dead[t].pop()
-  else:
-    inc symIdgen
-    result = SymId(symIdgen)
-    m.inScope.add result
-  m.live[result] = (k, t)
-
-proc allocTemp*(m: var SlotManager; t: TypeId; symIdgen: var int32): SymId {.inline.} =
-  result = allocRaw(m, t, ReuseTemps, Temp, symIdgen)
-
-proc allocVar*(m: var SlotManager; t: TypeId; symIdgen: var int32): SymId {.inline.} =
-  result = allocRaw(m, t, ReuseVars, Perm, symIdgen)
-
-proc freeLoc*(m: var SlotManager; s: SymId) =
-  let t = m.live.getOrDefault(s)
-  assert t[1].int != 0
-  m.live.del s
-  m.dead.mgetOrPut(t[1], @[]).add s
-
-proc freeTemp*(m: var SlotManager; s: SymId) =
-  let t = m.live.getOrDefault(s)
-  if t[1].int != 0 and t[0] == Temp:
-    m.live.del s
-    m.dead.mgetOrPut(t[1], @[]).add s
-
-iterator stillAlive*(m: SlotManager): (SymId, TypeId) =
-  for k, v in pairs(m.live):
-    yield (k, v[1])
-
-proc getType*(m: SlotManager; s: SymId): TypeId {.inline.} = m.live[s][1]
-
-proc openScope*(m: var SlotManager) =
-  m.inScope.add SymId(-1) # add marker
-
-proc closeScope*(m: var SlotManager) =
-  var i = m.inScope.len - 1
-  while i >= 0:
-    if m.inScope[i] == SymId(-1):
-      m.inScope.setLen i
-      break
-    dec i
-
-when isMainModule:
-  var symIdgen: int32
-  var m = initSlotManager({ReuseTemps})
-
-  var g = initTypeGraph(Literals())
-
-  let a = g.openType ArrayTy
-  g.addBuiltinType Int8Id
-  g.addArrayLen 5
-  let finalArrayType = finishType(g, a)
-
-  let obj = g.openType ObjectDecl
-  g.addName "MyType"
-
-  g.addField "p", finalArrayType, 0
-  let objB = finishType(g, obj)
-
-  let x = m.allocTemp(objB, symIdgen)
-  assert x.int == 0
-
-  let y = m.allocTemp(objB, symIdgen)
-  assert y.int == 1
-
-  let z = m.allocTemp(Int8Id, symIdgen)
-  assert z.int == 2
-
-  m.freeLoc y
-  let y2 = m.allocTemp(objB, symIdgen)
-  assert y2.int == 1

compiler/nir/nirtypes.nim

@@ -1,475 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Type system for NIR. Close to C's type system but without its quirks.
-
-import std / [assertions, hashes]
-import .. / ic / [bitabs, rodfiles]
-
-type
-  NirTypeKind* = enum
-    VoidTy, IntTy, UIntTy, FloatTy, BoolTy, CharTy, NameVal,
-    IntVal, SizeVal, AlignVal, OffsetVal,
-    AnnotationVal,
-    ObjectTy,
-    UnionTy,
-    VarargsTy, # the `...` in a C prototype; also the last "atom"
-    APtrTy, # pointer to aliasable memory
-    UPtrTy, # pointer to unique/unaliasable memory
-    AArrayPtrTy, # pointer to array of aliasable memory
-    UArrayPtrTy, # pointer to array of unique/unaliasable memory
-    ArrayTy,
-    LastArrayTy, # array of unspecified size as a last field inside an object
-    ProcTy,
-    ObjectDecl,
-    UnionDecl,
-    FieldDecl
-
-const
-  TypeKindBits = 8'u32
-  TypeKindMask = (1'u32 shl TypeKindBits) - 1'u32
-
-type
-  TypeNode* = object     # 4 bytes
-    x: uint32
-
-template kind*(n: TypeNode): NirTypeKind = NirTypeKind(n.x and TypeKindMask)
-template operand(n: TypeNode): uint32 = (n.x shr TypeKindBits)
-
-proc integralBits*(n: TypeNode): int {.inline.} =
-  # Number of bits in the IntTy, etc. Only valid for integral types.
-  assert n.kind in {IntTy, UIntTy, FloatTy, BoolTy, CharTy}
-  result = int(n.operand)
-
-template toX(k: NirTypeKind; operand: uint32): uint32 =
-  uint32(k) or (operand shl TypeKindBits)
-
-template toX(k: NirTypeKind; operand: LitId): uint32 =
-  uint32(k) or (operand.uint32 shl TypeKindBits)
-
-type
-  TypeId* = distinct int
-
-proc `==`*(a, b: TypeId): bool {.borrow.}
-proc hash*(a: TypeId): Hash {.borrow.}
-
-type
-  Literals* = ref object
-    strings*: BiTable[string]
-    numbers*: BiTable[int64]
-
-  TypeGraph* = object
-    nodes: seq[TypeNode]
-    lit: Literals
-
-const
-  VoidId* = TypeId 0
-  Bool8Id* = TypeId 1
-  Char8Id* = TypeId 2
-  Int8Id* = TypeId 3
-  Int16Id* = TypeId 4
-  Int32Id* = TypeId 5
-  Int64Id* = TypeId 6
-  UInt8Id* = TypeId 7
-  UInt16Id* = TypeId 8
-  UInt32Id* = TypeId 9
-  UInt64Id* = TypeId 10
-  Float32Id* = TypeId 11
-  Float64Id* = TypeId 12
-  VoidPtrId* = TypeId 13
-  LastBuiltinId* = 13
-
-proc initTypeGraph*(lit: Literals): TypeGraph =
-  result = TypeGraph(nodes: @[
-    TypeNode(x: toX(VoidTy, 0'u32)),
-    TypeNode(x: toX(BoolTy, 8'u32)),
-    TypeNode(x: toX(CharTy, 8'u32)),
-    TypeNode(x: toX(IntTy, 8'u32)),
-    TypeNode(x: toX(IntTy, 16'u32)),
-    TypeNode(x: toX(IntTy, 32'u32)),
-    TypeNode(x: toX(IntTy, 64'u32)),
-    TypeNode(x: toX(UIntTy, 8'u32)),
-    TypeNode(x: toX(UIntTy, 16'u32)),
-    TypeNode(x: toX(UIntTy, 32'u32)),
-    TypeNode(x: toX(UIntTy, 64'u32)),
-    TypeNode(x: toX(FloatTy, 32'u32)),
-    TypeNode(x: toX(FloatTy, 64'u32)),
-    TypeNode(x: toX(APtrTy, 2'u32)),
-    TypeNode(x: toX(VoidTy, 0'u32))
-  ], lit: lit)
-  assert result.nodes.len == LastBuiltinId+2
-
-type
-  TypePatchPos* = distinct int
-
-const
-  InvalidTypePatchPos* = TypePatchPos(-1)
-  LastAtomicValue = VarargsTy
-
-proc isValid(p: TypePatchPos): bool {.inline.} = p.int != -1
-
-proc prepare(tree: var TypeGraph; kind: NirTypeKind): TypePatchPos =
-  result = TypePatchPos tree.nodes.len
-  tree.nodes.add TypeNode(x: toX(kind, 1'u32))
-
-proc isAtom(tree: TypeGraph; pos: int): bool {.inline.} = tree.nodes[pos].kind <= LastAtomicValue
-proc isAtom(tree: TypeGraph; pos: TypeId): bool {.inline.} = tree.nodes[pos.int].kind <= LastAtomicValue
-
-proc patch(tree: var TypeGraph; pos: TypePatchPos) =
-  let pos = pos.int
-  let k = tree.nodes[pos].kind
-  assert k > LastAtomicValue
-  let distance = int32(tree.nodes.len - pos)
-  assert distance > 0
-  tree.nodes[pos].x = toX(k, cast[uint32](distance))
-
-proc len*(tree: TypeGraph): int {.inline.} = tree.nodes.len
-
-template rawSpan(n: TypeNode): int = int(operand(n))
-
-proc nextChild(tree: TypeGraph; pos: var int) {.inline.} =
-  if tree.nodes[pos].kind > LastAtomicValue:
-    assert tree.nodes[pos].operand > 0'u32
-    inc pos, tree.nodes[pos].rawSpan
-  else:
-    inc pos
-
-iterator sons*(tree: TypeGraph; n: TypeId): TypeId =
-  var pos = n.int
-  assert tree.nodes[pos].kind > LastAtomicValue
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  while pos < last:
-    yield TypeId pos
-    nextChild tree, pos
-
-template `[]`*(t: TypeGraph; n: TypeId): TypeNode = t.nodes[n.int]
-
-proc elementType*(tree: TypeGraph; n: TypeId): TypeId {.inline.} =
-  assert tree[n].kind in {APtrTy, UPtrTy, AArrayPtrTy, UArrayPtrTy, ArrayTy, LastArrayTy}
-  result = TypeId(n.int+1)
-
-proc litId*(n: TypeNode): LitId {.inline.} =
-  assert n.kind in {NameVal, IntVal, SizeVal, AlignVal, OffsetVal, AnnotationVal, ObjectTy, UnionTy}
-  result = LitId(n.operand)
-
-proc kind*(tree: TypeGraph; n: TypeId): NirTypeKind {.inline.} = tree[n].kind
-
-proc span(tree: TypeGraph; pos: int): int {.inline.} =
-  if tree.nodes[pos].kind <= LastAtomicValue: 1 else: int(tree.nodes[pos].operand)
-
-proc sons2(tree: TypeGraph; n: TypeId): (TypeId, TypeId) =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  result = (TypeId a, TypeId b)
-
-proc sons3(tree: TypeGraph; n: TypeId): (TypeId, TypeId, TypeId) =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  result = (TypeId a, TypeId b, TypeId c)
-
-proc arrayName*(tree: TypeGraph; n: TypeId): TypeId {.inline.} =
-  assert tree[n].kind == ArrayTy
-  let (_, _, c) = sons3(tree, n)
-  result = c
-
-proc arrayLen*(tree: TypeGraph; n: TypeId): BiggestInt =
-  assert tree[n].kind == ArrayTy
-  let (_, b) = sons2(tree, n)
-  result = tree.lit.numbers[LitId tree[b].operand]
-
-proc returnType*(tree: TypeGraph; n: TypeId): (TypeId, TypeId) =
-  # Returns the positions of the return type + calling convention.
-  var pos = n.int
-  assert tree.nodes[pos].kind == ProcTy
-  let a = n.int+1
-  let b = a + span(tree, a)
-  result = (TypeId b, TypeId a) # not a typo, order is reversed
-
-iterator params*(tree: TypeGraph; n: TypeId): TypeId =
-  var pos = n.int
-  assert tree.nodes[pos].kind == ProcTy
-  let last = pos + tree.nodes[pos].rawSpan
-  inc pos
-  nextChild tree, pos
-  nextChild tree, pos
-  while pos < last:
-    yield TypeId pos
-    nextChild tree, pos
-
-proc openType*(tree: var TypeGraph; kind: NirTypeKind): TypePatchPos =
-  assert kind in {APtrTy, UPtrTy, AArrayPtrTy, UArrayPtrTy,
-    ArrayTy, LastArrayTy, ProcTy, ObjectDecl, UnionDecl,
-    FieldDecl}
-  result = prepare(tree, kind)
-
-template typeInvariant(p: TypePatchPos) =
-  when false:
-    if tree[TypeId(p)].kind == FieldDecl:
-      var k = 0
-      for ch in sons(tree, TypeId(p)):
-        inc k
-      assert k > 2, "damn! " & $k
-
-proc sealType*(tree: var TypeGraph; p: TypePatchPos) =
-  patch tree, p
-  typeInvariant(p)
-
-proc finishType*(tree: var TypeGraph; p: TypePatchPos): TypeId =
-  # Search for an existing instance of this type in
-  # order to reduce memory consumption:
-  patch tree, p
-  typeInvariant(p)
-
-  let s = span(tree, p.int)
-  var i = 0
-  while i < p.int:
-    if tree.nodes[i].x == tree.nodes[p.int].x:
-      var isMatch = true
-      for j in 1..<s:
-        if tree.nodes[j+i].x == tree.nodes[j+p.int].x:
-          discard "still a match"
-        else:
-          isMatch = false
-          break
-      if isMatch:
-        if p.int+s == tree.len:
-          setLen tree.nodes, p.int
-        return TypeId(i)
-    nextChild tree, i
-  result = TypeId(p)
-
-proc nominalType*(tree: var TypeGraph; kind: NirTypeKind; name: string): TypeId =
-  assert kind in {ObjectTy, UnionTy}
-  let content = TypeNode(x: toX(kind, tree.lit.strings.getOrIncl(name)))
-  for i in 0..<tree.len:
-    if tree.nodes[i].x == content.x:
-      return TypeId(i)
-  result = TypeId tree.nodes.len
-  tree.nodes.add content
-
-proc addNominalType*(tree: var TypeGraph; kind: NirTypeKind; name: string) =
-  assert kind in {ObjectTy, UnionTy}
-  tree.nodes.add TypeNode(x: toX(kind, tree.lit.strings.getOrIncl(name)))
-
-proc getTypeTag*(tree: TypeGraph; t: TypeId): string =
-  assert tree[t].kind in {ObjectTy, UnionTy}
-  result = tree.lit.strings[LitId tree[t].operand]
-
-proc addVarargs*(tree: var TypeGraph) =
-  tree.nodes.add TypeNode(x: toX(VarargsTy, 0'u32))
-
-proc getFloat128Type*(tree: var TypeGraph): TypeId =
-  result = TypeId tree.nodes.len
-  tree.nodes.add TypeNode(x: toX(FloatTy, 128'u32))
-
-proc addBuiltinType*(g: var TypeGraph; id: TypeId) =
-  g.nodes.add g[id]
-
-template firstSon*(n: TypeId): TypeId = TypeId(n.int+1)
-
-proc addType*(g: var TypeGraph; t: TypeId) =
-  # We cannot simply copy `*Decl` nodes. We have to introduce `*Ty` nodes instead:
-  if g[t].kind in {ObjectDecl, UnionDecl}:
-    assert g[t.firstSon].kind == NameVal
-    let name = LitId g[t.firstSon].operand
-    if g[t].kind == ObjectDecl:
-      g.nodes.add TypeNode(x: toX(ObjectTy, name))
-    else:
-      g.nodes.add TypeNode(x: toX(UnionTy, name))
-  else:
-    let pos = t.int
-    let L = span(g, pos)
-    let d = g.nodes.len
-    g.nodes.setLen(d + L)
-    assert L > 0
-    for i in 0..<L:
-      g.nodes[d+i] = g.nodes[pos+i]
-
-proc addArrayLen*(g: var TypeGraph; len: int64) =
-  g.nodes.add TypeNode(x: toX(IntVal, g.lit.numbers.getOrIncl(len)))
-
-proc addSize*(g: var TypeGraph; s: int64) =
-  g.nodes.add TypeNode(x: toX(SizeVal, g.lit.numbers.getOrIncl(s)))
-
-proc addOffset*(g: var TypeGraph; offset: int64) =
-  g.nodes.add TypeNode(x: toX(OffsetVal, g.lit.numbers.getOrIncl(offset)))
-
-proc addAlign*(g: var TypeGraph; a: int64) =
-  g.nodes.add TypeNode(x: toX(AlignVal, g.lit.numbers.getOrIncl(a)))
-
-proc addName*(g: var TypeGraph; name: string) =
-  g.nodes.add TypeNode(x: toX(NameVal, g.lit.strings.getOrIncl(name)))
-
-proc addAnnotation*(g: var TypeGraph; name: string) =
-  g.nodes.add TypeNode(x: toX(NameVal, g.lit.strings.getOrIncl(name)))
-
-proc addField*(g: var TypeGraph; name: string; typ: TypeId; offset: int64) =
-  let f = g.openType FieldDecl
-  g.addType typ
-  g.addOffset offset
-  g.addName name
-  sealType(g, f)
-
-proc ptrTypeOf*(g: var TypeGraph; t: TypeId): TypeId =
-  let f = g.openType APtrTy
-  g.addType t
-  result = finishType(g, f)
-
-proc arrayPtrTypeOf*(g: var TypeGraph; t: TypeId): TypeId =
-  let f = g.openType AArrayPtrTy
-  g.addType t
-  result = finishType(g, f)
-
-proc store*(r: var RodFile; g: TypeGraph) =
-  storeSeq r, g.nodes
-
-proc load*(r: var RodFile; g: var TypeGraph) =
-  loadSeq r, g.nodes
-
-proc toString*(dest: var string; g: TypeGraph; i: TypeId) =
-  case g[i].kind
-  of VoidTy: dest.add "void"
-  of IntTy:
-    dest.add "i"
-    dest.addInt g[i].operand
-  of UIntTy:
-    dest.add "u"
-    dest.addInt g[i].operand
-  of FloatTy:
-    dest.add "f"
-    dest.addInt g[i].operand
-  of BoolTy:
-    dest.add "b"
-    dest.addInt g[i].operand
-  of CharTy:
-    dest.add "c"
-    dest.addInt g[i].operand
-  of NameVal, AnnotationVal:
-    dest.add g.lit.strings[LitId g[i].operand]
-  of IntVal, SizeVal, AlignVal, OffsetVal:
-    dest.add $g[i].kind
-    dest.add ' '
-    dest.add $g.lit.numbers[LitId g[i].operand]
-  of VarargsTy:
-    dest.add "..."
-  of APtrTy:
-    dest.add "aptr["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of UPtrTy:
-    dest.add "uptr["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of AArrayPtrTy:
-    dest.add "aArrayPtr["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of UArrayPtrTy:
-    dest.add "uArrayPtr["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of ArrayTy:
-    dest.add "Array["
-    let (elems, len, name) = g.sons3(i)
-    toString(dest, g, elems)
-    dest.add ", "
-    toString(dest, g, len)
-    dest.add ", "
-    toString(dest, g, name)
-    dest.add "]"
-  of LastArrayTy:
-    # array of unspecified size as a last field inside an object
-    dest.add "LastArrayTy["
-    toString(dest, g, g.elementType(i))
-    dest.add "]"
-  of ObjectTy:
-    dest.add "object "
-    dest.add g.lit.strings[LitId g[i].operand]
-  of UnionTy:
-    dest.add "union "
-    dest.add g.lit.strings[LitId g[i].operand]
-  of ProcTy:
-    dest.add "proc["
-    for t in sons(g, i):
-      dest.add ' '
-      toString(dest, g, t)
-    dest.add "]"
-  of ObjectDecl:
-    dest.add "object["
-    for t in sons(g, i):
-      toString(dest, g, t)
-      dest.add '\n'
-    dest.add "]"
-  of UnionDecl:
-    dest.add "union["
-    for t in sons(g, i):
-      toString(dest, g, t)
-      dest.add '\n'
-    dest.add "]"
-  of FieldDecl:
-    dest.add "field["
-    for t in sons(g, i):
-      toString(dest, g, t)
-      dest.add ' '
-    dest.add "]"
-
-    when false:
-      let (typ, offset, name) = g.sons3(i)
-      toString(dest, g, typ)
-      dest.add ' '
-      toString(dest, g, offset)
-      dest.add ' '
-      toString(dest, g, name)
-
-proc toString*(dest: var string; g: TypeGraph) =
-  var i = 0
-  while i < g.len:
-    dest.add "T<"
-    dest.addInt i
-    dest.add "> "
-    toString(dest, g, TypeId i)
-    dest.add '\n'
-    nextChild g, i
-
-iterator allTypes*(g: TypeGraph; start = 0): TypeId =
-  var i = start
-  while i < g.len:
-    yield TypeId i
-    nextChild g, i
-
-iterator allTypesIncludingInner*(g: TypeGraph; start = 0): TypeId =
-  var i = start
-  while i < g.len:
-    yield TypeId i
-    inc i
-
-proc `$`(g: TypeGraph): string =
-  result = ""
-  toString(result, g)
-
-when isMainModule:
-  var g = initTypeGraph(Literals())
-
-  let a = g.openType ArrayTy
-  g.addBuiltinType Int8Id
-  g.addArrayLen 5
-  g.addName "SomeArray"
-  let finalArrayType = finishType(g, a)
-
-  let obj = g.openType ObjectDecl
-  g.nodes.add TypeNode(x: toX(NameVal, g.lit.strings.getOrIncl("MyType")))
-
-  g.addField "p", finalArrayType, 0
-  sealType(g, obj)
-
-  echo g

compiler/nir/stringcases.nim

@@ -1,200 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## included from ast2ir.nim
-
-#[
-
-case s
-of "abc", "abbd":
-  echo 1
-of "hah":
-  echo 2
-of "gah":
-  echo 3
-
-# we produce code like this:
-
-if s[0] <= 'a':
-  if s == "abc: goto L1
-  elif s == "abbd": goto L1
-else:
-  if s[2] <= 'h':
-    if s == "hah": goto L2
-    elif s == "gah": goto L3
-goto afterCase
-
-L1:
-  echo 1
-  goto afterCase
-L2:
-  echo 2
-  goto afterCase
-L3:
-  echo 3
-  goto afterCase
-
-afterCase: ...
-
-]#
-
-# We split the set of strings into 2 sets of roughly the same size.
-# The condition used for splitting is a (position, char) tuple.
-# Every string of length > position for which s[position] <= char is in one
-# set else it is in the other set.
-
-from std/sequtils import addUnique
-
-type
-  Key = (LitId, LabelId)
-
-proc splitValue(strings: BiTable[string]; a: openArray[Key]; position: int): (char, float) =
-  var cand: seq[char] = @[]
-  for t in items a:
-    let s = strings[t[0]]
-    if s.len > position: cand.addUnique s[position]
-
-  result = ('\0', -1.0)
-  for disc in items cand:
-    var hits = 0
-    for t in items a:
-      let s = strings[t[0]]
-      if s.len > position and s[position] <= disc:
-        inc hits
-    # the split is the better, the more `hits` is close to `a.len / 2`:
-    let grade = 100000.0 - abs(hits.float - a.len.float / 2.0)
-    if grade > result[1]:
-      result = (disc, grade)
-
-proc tryAllPositions(strings: BiTable[string]; a: openArray[Key]): (char, int) =
-  var m = 0
-  for t in items a:
-    m = max(m, strings[t[0]].len)
-
-  result = ('\0', -1)
-  var best = -1.0
-  for i in 0 ..< m:
-    let current = splitValue(strings, a, i)
-    if current[1] > best:
-      best = current[1]
-      result = (current[0], i)
-
-type
-  SearchKind = enum
-    LinearSearch, SplitSearch
-  SearchResult* = object
-    case kind: SearchKind
-    of LinearSearch:
-      a: seq[Key]
-    of SplitSearch:
-      span: int
-      best: (char, int)
-
-proc emitLinearSearch(strings: BiTable[string]; a: openArray[Key]; dest: var seq[SearchResult]) =
-  var d = SearchResult(kind: LinearSearch, a: @[])
-  for x in a: d.a.add x
-  dest.add d
-
-proc split(strings: BiTable[string]; a: openArray[Key]; dest: var seq[SearchResult]) =
-  if a.len <= 4:
-    emitLinearSearch strings, a, dest
-  else:
-    let best = tryAllPositions(strings, a)
-    var groupA: seq[Key] = @[]
-    var groupB: seq[Key] = @[]
-    for t in items a:
-      let s = strings[t[0]]
-      if s.len > best[1] and s[best[1]] <= best[0]:
-        groupA.add t
-      else:
-        groupB.add t
-    if groupA.len == 0 or groupB.len == 0:
-      emitLinearSearch strings, a, dest
-    else:
-      let toPatch = dest.len
-      dest.add SearchResult(kind: SplitSearch, span: 1, best: best)
-      split strings, groupA, dest
-      split strings, groupB, dest
-      let dist = dest.len - toPatch
-      assert dist > 0
-      dest[toPatch].span = dist
-
-proc toProblemDescription(c: var ProcCon; n: PNode): (seq[Key], LabelId) =
-  result = (@[], newLabels(c.labelGen, n.len))
-  assert n.kind == nkCaseStmt
-  for i in 1..<n.len:
-    let it = n[i]
-    let thisBranch = LabelId(result[1].int + i - 1)
-    if it.kind == nkOfBranch:
-      for j in 0..<it.len-1:
-        assert it[j].kind in {nkStrLit..nkTripleStrLit}
-        result[0].add (c.lit.strings.getOrIncl(it[j].strVal), thisBranch)
-
-proc decodeSolution(c: var ProcCon; dest: var Tree; s: seq[SearchResult]; i: int;
-                    selector: Value; info: PackedLineInfo) =
-  case s[i].kind
-  of SplitSearch:
-    let thenA = i+1
-    let elseA = thenA + (if s[thenA].kind == LinearSearch: 1 else: s[thenA].span)
-    let best = s[i].best
-
-    let tmp = getTemp(c, Bool8Id, info)
-    buildTyped dest, info, Asgn, Bool8Id:
-      dest.copyTree tmp
-      buildTyped dest, info, Call, Bool8Id:
-        c.addUseCodegenProc dest, "nimStrAtLe", info
-        dest.copyTree selector
-        dest.addIntVal c.lit.numbers, info, c.m.nativeIntId, best[1]
-        dest.addIntVal c.lit.numbers, info, Char8Id, best[0].int
-
-    template then() =
-      c.decodeSolution dest, s, thenA, selector, info
-    template otherwise() =
-      c.decodeSolution dest, s, elseA, selector, info
-    buildIfThenElse tmp, then, otherwise
-    freeTemp c, tmp
-
-  of LinearSearch:
-    let tmp = getTemp(c, Bool8Id, info)
-    for x in s[i].a:
-      buildTyped dest, info, Asgn, Bool8Id:
-        dest.copyTree tmp
-        buildTyped dest, info, Call, Bool8Id:
-          c.addUseCodegenProc dest, "eqStrings", info
-          dest.copyTree selector
-          dest.addStrLit info, x[0]
-      buildIf tmp:
-        c.code.gotoLabel info, Goto, x[1]
-    freeTemp c, tmp
-
-proc genStringCase(c: var ProcCon; n: PNode; d: var Value) =
-  let (problem, firstBranch) = toProblemDescription(c, n)
-  var solution: seq[SearchResult] = @[]
-  split c.lit.strings, problem, solution
-
-  # XXX Todo move complex case selector into a temporary.
-  let selector = c.genx(n[0])
-
-  let info = toLineInfo(c, n.info)
-  decodeSolution c, c.code, solution, 0, selector, info
-
-  let lend = newLabel(c.labelGen)
-  c.code.addLabel info, Goto, lend
-  for i in 1..<n.len:
-    let it = n[i]
-    let thisBranch = LabelId(firstBranch.int + i - 1)
-    c.code.addLabel info, Label, thisBranch
-    if it.kind == nkOfBranch:
-      gen(c, it.lastSon, d)
-      c.code.addLabel info, Goto, lend
-    else:
-      gen(c, it.lastSon, d)
-
-  c.code.addLabel info, Label, lend
-  freeTemp c, selector

compiler/nir/types2ir.nim

@@ -1,525 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2023 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import std / [assertions, tables, sets]
-import ".." / [ast, types, options, sighashes, modulegraphs]
-import nirtypes
-
-type
-  TypesCon* = object
-    processed: Table[ItemId, TypeId]
-    processedByName: Table[string, TypeId]
-    recursionCheck: HashSet[ItemId]
-    conf: ConfigRef
-    stringType: TypeId
-
-proc initTypesCon*(conf: ConfigRef): TypesCon =
-  TypesCon(conf: conf, stringType: TypeId(-1))
-
-proc mangle(c: var TypesCon; t: PType): string =
-  result = $sighashes.hashType(t, c.conf)
-
-template cached(c: var TypesCon; t: PType; body: untyped) =
-  result = c.processed.getOrDefault(t.itemId)
-  if result.int == 0:
-    body
-    c.processed[t.itemId] = result
-
-template cachedByName(c: var TypesCon; t: PType; body: untyped) =
-  let key = mangle(c, t)
-  result = c.processedByName.getOrDefault(key)
-  if result.int == 0:
-    body
-    c.processedByName[key] = result
-
-proc typeToIr*(c: var TypesCon; g: var TypeGraph; t: PType): TypeId
-
-proc collectFieldTypes(c: var TypesCon; g: var TypeGraph; n: PNode; dest: var Table[ItemId, TypeId]) =
-  case n.kind
-  of nkRecList:
-    for i in 0..<n.len:
-      collectFieldTypes(c, g, n[i], dest)
-  of nkRecCase:
-    assert(n[0].kind == nkSym)
-    collectFieldTypes(c, g, n[0], dest)
-    for i in 1..<n.len:
-      case n[i].kind
-      of nkOfBranch, nkElse:
-        collectFieldTypes c, g, lastSon(n[i]), dest
-      else: discard
-  of nkSym:
-    dest[n.sym.itemId] = typeToIr(c, g, n.sym.typ)
-  else:
-    assert false, "unknown node kind: " & $n.kind
-
-proc objectToIr(c: var TypesCon; g: var TypeGraph; n: PNode; fieldTypes: Table[ItemId, TypeId]; unionId: var int) =
-  case n.kind
-  of nkRecList:
-    for i in 0..<n.len:
-      objectToIr(c, g, n[i], fieldTypes, unionId)
-  of nkRecCase:
-    assert(n[0].kind == nkSym)
-    objectToIr(c, g, n[0], fieldTypes, unionId)
-    let u = openType(g, UnionDecl)
-    g.addName "u_" & $unionId
-    inc unionId
-    for i in 1..<n.len:
-      case n[i].kind
-      of nkOfBranch, nkElse:
-        let subObj = openType(g, ObjectDecl)
-        g.addName "uo_" & $unionId & "_" & $i
-        objectToIr c, g, lastSon(n[i]), fieldTypes, unionId
-        sealType(g, subObj)
-      else: discard
-    sealType(g, u)
-  of nkSym:
-    g.addField n.sym.name.s & "_" & $n.sym.position, fieldTypes[n.sym.itemId], n.sym.offset
-  else:
-    assert false, "unknown node kind: " & $n.kind
-
-proc objectToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  if t.baseClass != nil:
-    # ensure we emitted the base type:
-    discard typeToIr(c, g, t.baseClass)
-
-  var unionId = 0
-  var fieldTypes = initTable[ItemId, TypeId]()
-  collectFieldTypes c, g, t.n, fieldTypes
-  let obj = openType(g, ObjectDecl)
-  g.addName mangle(c, t)
-  g.addSize c.conf.getSize(t)
-  g.addAlign c.conf.getAlign(t)
-
-  if t.baseClass != nil:
-    g.addNominalType(ObjectTy, mangle(c, t.baseClass))
-  else:
-    g.addBuiltinType VoidId # object does not inherit
-    if not lacksMTypeField(t):
-      let f2 = g.openType FieldDecl
-      let voidPtr = openType(g, APtrTy)
-      g.addBuiltinType(VoidId)
-      sealType(g, voidPtr)
-      g.addOffset 0 # type field is always at offset 0
-      g.addName "m_type"
-      sealType(g, f2) # FieldDecl
-
-  objectToIr c, g, t.n, fieldTypes, unionId
-  result = finishType(g, obj)
-
-proc objectHeaderToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  result = g.nominalType(ObjectTy, mangle(c, t))
-
-proc tupleToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  var fieldTypes = newSeq[TypeId](t.len)
-  for i in 0..<t.len:
-    fieldTypes[i] = typeToIr(c, g, t[i])
-  let obj = openType(g, ObjectDecl)
-  g.addName mangle(c, t)
-  g.addSize c.conf.getSize(t)
-  g.addAlign c.conf.getAlign(t)
-
-  var accum = OffsetAccum(maxAlign: 1)
-  for i in 0..<t.len:
-    let child = t[i]
-    g.addField "f_" & $i, fieldTypes[i], accum.offset
-
-    computeSizeAlign(c.conf, child)
-    accum.align(child.align)
-    accum.inc(int32(child.size))
-  result = finishType(g, obj)
-
-proc procToIr(c: var TypesCon; g: var TypeGraph; t: PType; addEnv = false): TypeId =
-  var fieldTypes = newSeq[TypeId](0)
-  for i in 0..<t.len:
-    if t[i] == nil or not isCompileTimeOnly(t[i]):
-      fieldTypes.add typeToIr(c, g, t[i])
-  let obj = openType(g, ProcTy)
-
-  case t.callConv
-  of ccNimCall, ccFastCall, ccClosure: g.addAnnotation "__fastcall"
-  of ccStdCall: g.addAnnotation "__stdcall"
-  of ccCDecl: g.addAnnotation "__cdecl"
-  of ccSafeCall: g.addAnnotation "__safecall"
-  of ccSysCall: g.addAnnotation "__syscall"
-  of ccInline: g.addAnnotation "__inline"
-  of ccNoInline: g.addAnnotation "__noinline"
-  of ccThisCall: g.addAnnotation "__thiscall"
-  of ccNoConvention, ccMember: g.addAnnotation ""
-
-  for i in 0..<fieldTypes.len:
-    g.addType fieldTypes[i]
-
-  if addEnv:
-    let a = openType(g, APtrTy)
-    g.addBuiltinType(VoidId)
-    sealType(g, a)
-
-  if tfVarargs in t.flags:
-    g.addVarargs()
-  result = finishType(g, obj)
-
-proc nativeInt(c: TypesCon): TypeId =
-  case c.conf.target.intSize
-  of 2: result = Int16Id
-  of 4: result = Int32Id
-  else: result = Int64Id
-
-proc openArrayPayloadType*(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  let e = elementType(t)
-  let elementType = typeToIr(c, g, e)
-  let arr = g.openType AArrayPtrTy
-  g.addType elementType
-  result = finishType(g, arr) # LastArrayTy
-
-proc openArrayToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  # object (a: ArrayPtr[T], len: int)
-  let e = elementType(t)
-  let mangledBase = mangle(c, e)
-  let typeName = "NimOpenArray" & mangledBase
-
-  let elementType = typeToIr(c, g, e)
-  #assert elementType.int >= 0, typeToString(t)
-
-  let p = openType(g, ObjectDecl)
-  g.addName typeName
-  g.addSize c.conf.target.ptrSize*2
-  g.addAlign c.conf.target.ptrSize
-
-  let f = g.openType FieldDecl
-  let arr = g.openType AArrayPtrTy
-  g.addType elementType
-  sealType(g, arr) # LastArrayTy
-  g.addOffset 0
-  g.addName "data"
-  sealType(g, f) # FieldDecl
-
-  g.addField "len", c.nativeInt, c.conf.target.ptrSize
-
-  result = finishType(g, p) # ObjectDecl
-
-proc strPayloadType(c: var TypesCon; g: var TypeGraph): (string, TypeId) =
-  result = ("NimStrPayload", TypeId(-1))
-  let p = openType(g, ObjectDecl)
-  g.addName result[0]
-  g.addSize c.conf.target.ptrSize*2
-  g.addAlign c.conf.target.ptrSize
-
-  g.addField "cap", c.nativeInt, 0
-
-  let f = g.openType FieldDecl
-  let arr = g.openType LastArrayTy
-  g.addBuiltinType Char8Id
-  result[1] = finishType(g, arr) # LastArrayTy
-  g.addOffset c.conf.target.ptrSize # comes after the len field
-  g.addName "data"
-  sealType(g, f) # FieldDecl
-
-  sealType(g, p)
-
-proc strPayloadPtrType*(c: var TypesCon; g: var TypeGraph): (TypeId, TypeId) =
-  let (mangled, arrayType) = strPayloadType(c, g)
-  let ffp = g.openType APtrTy
-  g.addNominalType ObjectTy, mangled
-  result = (finishType(g, ffp), arrayType) # APtrTy
-
-proc stringToIr(c: var TypesCon; g: var TypeGraph): TypeId =
-  #[
-
-    NimStrPayload = object
-      cap: int
-      data: UncheckedArray[char]
-
-    NimStringV2 = object
-      len: int
-      p: ptr NimStrPayload
-
-  ]#
-  let payload = strPayloadType(c, g)
-
-  let str = openType(g, ObjectDecl)
-  g.addName "NimStringV2"
-  g.addSize c.conf.target.ptrSize*2
-  g.addAlign c.conf.target.ptrSize
-
-  g.addField "len", c.nativeInt, 0
-
-  let fp = g.openType FieldDecl
-  let ffp = g.openType APtrTy
-  g.addNominalType ObjectTy, "NimStrPayload"
-  sealType(g, ffp) # APtrTy
-  g.addOffset c.conf.target.ptrSize # comes after 'len' field
-  g.addName "p"
-  sealType(g, fp) # FieldDecl
-
-  result = finishType(g, str) # ObjectDecl
-
-proc seqPayloadType(c: var TypesCon; g: var TypeGraph; t: PType): (string, TypeId) =
-  #[
-    NimSeqPayload[T] = object
-      cap: int
-      data: UncheckedArray[T]
-  ]#
-  let e = elementType(t)
-  result = (mangle(c, e), TypeId(-1))
-  let payloadName = "NimSeqPayload" & result[0]
-
-  let elementType = typeToIr(c, g, e)
-
-  let p = openType(g, ObjectDecl)
-  g.addName payloadName
-  g.addSize c.conf.target.intSize
-  g.addAlign c.conf.target.intSize
-
-  g.addField "cap", c.nativeInt, 0
-
-  let f = g.openType FieldDecl
-  let arr = g.openType LastArrayTy
-  g.addType elementType
-  # DO NOT USE `finishType` here as it is an inner type. This is subtle and we
-  # probably need an even better API here.
-  sealType(g, arr)
-  result[1] = TypeId(arr)
-
-  g.addOffset c.conf.target.ptrSize
-  g.addName "data"
-  sealType(g, f) # FieldDecl
-
-  sealType(g, p)
-
-proc seqPayloadPtrType*(c: var TypesCon; g: var TypeGraph; t: PType): (TypeId, TypeId) =
-  let (mangledBase, arrayType) = seqPayloadType(c, g, t)
-  let ffp = g.openType APtrTy
-  g.addNominalType ObjectTy, "NimSeqPayload" & mangledBase
-  result = (finishType(g, ffp), arrayType) # APtrTy
-
-proc seqToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  #[
-    NimSeqV2*[T] = object
-      len: int
-      p: ptr NimSeqPayload[T]
-  ]#
-  let (mangledBase, _) = seqPayloadType(c, g, t)
-
-  let sq = openType(g, ObjectDecl)
-  g.addName "NimSeqV2" & mangledBase
-  g.addSize c.conf.getSize(t)
-  g.addAlign c.conf.getAlign(t)
-
-  g.addField "len", c.nativeInt, 0
-
-  let fp = g.openType FieldDecl
-  let ffp = g.openType APtrTy
-  g.addNominalType ObjectTy, "NimSeqPayload" & mangledBase
-  sealType(g, ffp) # APtrTy
-  g.addOffset c.conf.target.ptrSize
-  g.addName "p"
-  sealType(g, fp) # FieldDecl
-
-  result = finishType(g, sq) # ObjectDecl
-
-
-proc closureToIr(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  # struct {fn(args, void* env), env}
-  # typedef struct {$n" &
-  #        "N_NIMCALL_PTR($2, ClP_0) $3;$n" &
-  #        "void* ClE_0;$n} $1;$n"
-  let mangledBase = mangle(c, t)
-  let typeName = "NimClosure" & mangledBase
-
-  let procType = procToIr(c, g, t, addEnv=true)
-
-  let p = openType(g, ObjectDecl)
-  g.addName typeName
-  g.addSize c.conf.getSize(t)
-  g.addAlign c.conf.getAlign(t)
-
-  let f = g.openType FieldDecl
-  g.addType procType
-  g.addOffset 0
-  g.addName "ClP_0"
-  sealType(g, f) # FieldDecl
-
-  let f2 = g.openType FieldDecl
-  let voidPtr = openType(g, APtrTy)
-  g.addBuiltinType(VoidId)
-  sealType(g, voidPtr)
-
-  g.addOffset c.conf.target.ptrSize
-  g.addName "ClE_0"
-  sealType(g, f2) # FieldDecl
-
-  result = finishType(g, p) # ObjectDecl
-
-proc bitsetBasetype*(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  let s = int(getSize(c.conf, t))
-  case s
-  of 1: result = UInt8Id
-  of 2: result = UInt16Id
-  of 4: result = UInt32Id
-  of 8: result = UInt64Id
-  else: result = UInt8Id
-
-proc typeToIr*(c: var TypesCon; g: var TypeGraph; t: PType): TypeId =
-  if t == nil: return VoidId
-  case t.kind
-  of tyInt:
-    case int(getSize(c.conf, t))
-    of 2: result = Int16Id
-    of 4: result = Int32Id
-    else: result = Int64Id
-  of tyInt8: result = Int8Id
-  of tyInt16: result = Int16Id
-  of tyInt32: result = Int32Id
-  of tyInt64: result = Int64Id
-  of tyFloat:
-    case int(getSize(c.conf, t))
-    of 4: result = Float32Id
-    else: result = Float64Id
-  of tyFloat32: result = Float32Id
-  of tyFloat64: result = Float64Id
-  of tyFloat128: result = getFloat128Type(g)
-  of tyUInt:
-    case int(getSize(c.conf, t))
-    of 2: result = UInt16Id
-    of 4: result = UInt32Id
-    else: result = UInt64Id
-  of tyUInt8: result = UInt8Id
-  of tyUInt16: result = UInt16Id
-  of tyUInt32: result = UInt32Id
-  of tyUInt64: result = UInt64Id
-  of tyBool: result = Bool8Id
-  of tyChar: result = Char8Id
-  of tyVoid: result = VoidId
-  of tySink, tyGenericInst, tyDistinct, tyAlias, tyOwned, tyRange:
-    result = typeToIr(c, g, t.skipModifier)
-  of tyEnum:
-    if firstOrd(c.conf, t) < 0:
-      result = Int32Id
-    else:
-      case int(getSize(c.conf, t))
-      of 1: result = UInt8Id
-      of 2: result = UInt16Id
-      of 4: result = Int32Id
-      of 8: result = Int64Id
-      else: result = Int32Id
-  of tyOrdinal, tyGenericBody, tyGenericParam, tyInferred, tyStatic:
-    if t.len > 0:
-      result = typeToIr(c, g, t.skipModifier)
-    else:
-      result = TypeId(-1)
-  of tyFromExpr:
-    if t.n != nil and t.n.typ != nil:
-      result = typeToIr(c, g, t.n.typ)
-    else:
-      result = TypeId(-1)
-  of tyArray:
-    cached(c, t):
-      var n = toInt64(lengthOrd(c.conf, t))
-      if n <= 0: n = 1   # make an array of at least one element
-      let elemType = typeToIr(c, g, t.elementType)
-      let a = openType(g, ArrayTy)
-      g.addType(elemType)
-      g.addArrayLen n
-      g.addName mangle(c, t)
-      result = finishType(g, a)
-  of tyPtr, tyRef:
-    cached(c, t):
-      let e = t.elementType
-      if e.kind == tyUncheckedArray:
-        let elemType = typeToIr(c, g, e.elementType)
-        let a = openType(g, AArrayPtrTy)
-        g.addType(elemType)
-        result = finishType(g, a)
-      else:
-        let elemType = typeToIr(c, g, t.elementType)
-        let a = openType(g, APtrTy)
-        g.addType(elemType)
-        result = finishType(g, a)
-  of tyVar, tyLent:
-    cached(c, t):
-      let e = t.elementType
-      if e.skipTypes(abstractInst).kind in {tyOpenArray, tyVarargs}:
-        # skip the modifier, `var openArray` is a (ptr, len) pair too:
-        result = typeToIr(c, g, e)
-      else:
-        let elemType = typeToIr(c, g, e)
-        let a = openType(g, APtrTy)
-        g.addType(elemType)
-        result = finishType(g, a)
-  of tySet:
-    let s = int(getSize(c.conf, t))
-    case s
-    of 1: result = UInt8Id
-    of 2: result = UInt16Id
-    of 4: result = UInt32Id
-    of 8: result = UInt64Id
-    else:
-      # array[U8, s]
-      cached(c, t):
-        let a = openType(g, ArrayTy)
-        g.addType(UInt8Id)
-        g.addArrayLen s
-        g.addName mangle(c, t)
-        result = finishType(g, a)
-  of tyPointer, tyNil:
-    # tyNil can happen for code like: `const CRAP = nil` which we have in posix.nim
-    let a = openType(g, APtrTy)
-    g.addBuiltinType(VoidId)
-    result = finishType(g, a)
-  of tyObject:
-    # Objects are special as they can be recursive in Nim. This is easily solvable.
-    # We check if we are already "processing" t. If so, we produce `ObjectTy`
-    # instead of `ObjectDecl`.
-    cached(c, t):
-      if not c.recursionCheck.containsOrIncl(t.itemId):
-        result = objectToIr(c, g, t)
-      else:
-        result = objectHeaderToIr(c, g, t)
-  of tyTuple:
-    cachedByName(c, t):
-      result = tupleToIr(c, g, t)
-  of tyProc:
-    cached(c, t):
-      if t.callConv == ccClosure:
-        result = closureToIr(c, g, t)
-      else:
-        result = procToIr(c, g, t)
-  of tyVarargs, tyOpenArray:
-    cached(c, t):
-      result = openArrayToIr(c, g, t)
-  of tyString:
-    if c.stringType.int < 0:
-      result = stringToIr(c, g)
-      c.stringType = result
-    else:
-      result = c.stringType
-  of tySequence:
-    cachedByName(c, t):
-      result = seqToIr(c, g, t)
-  of tyCstring:
-    cached(c, t):
-      let a = openType(g, AArrayPtrTy)
-      g.addBuiltinType Char8Id
-      result = finishType(g, a)
-  of tyUncheckedArray:
-    # We already handled the `ptr UncheckedArray` in a special way.
-    cached(c, t):
-      let elemType = typeToIr(c, g, t.elementType)
-      let a = openType(g, LastArrayTy)
-      g.addType(elemType)
-      result = finishType(g, a)
-  of tyUntyped, tyTyped:
-    # this avoids a special case for system.echo which is not a generic but
-    # uses `varargs[typed]`:
-    result = VoidId
-  of tyNone, tyEmpty, tyTypeDesc,
-     tyGenericInvocation, tyProxy, tyBuiltInTypeClass,
-     tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass,
-     tyAnd, tyOr, tyNot, tyAnything, tyConcept, tyIterable, tyForward:
-    result = TypeId(-1)

compiler/options.nim

@@ -139,7 +139,6 @@ type
     backendCpp = "cpp"
     backendJs = "js"
     backendObjc = "objc"
-    backendNir = "nir"
     # backendNimscript = "nimscript" # this could actually work
     # backendLlvm = "llvm" # probably not well supported; was cmdCompileToLLVM
 
@@ -147,7 +146,6 @@ type
     cmdNone # not yet processed command
     cmdUnknown # command unmapped
     cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS,
-    cmdCompileToNir,
     cmdCrun # compile and run in nimache
     cmdTcc # run the project via TCC backend
     cmdCheck # semantic checking for whole project
@@ -176,7 +174,7 @@ type
 
 const
   cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,
-                  cmdCompileToJS, cmdCrun, cmdCompileToNir}
+                  cmdCompileToJS, cmdCrun}
   cmdDocLike* = {cmdDoc0, cmdDoc, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
                  cmdCtags, cmdBuildindex}
 
@@ -246,13 +244,15 @@ type
     emitGenerics
       ## generics are emitted in the module that contains them.
       ## Useful for libraries that rely on local passC
+    jsNoLambdaLifting
+      ## Old transformation for closures in JS backend
 
   SymbolFilesOption* = enum
     disabledSf, writeOnlySf, readOnlySf, v2Sf, stressTest
 
   TSystemCC* = enum
     ccNone, ccGcc, ccNintendoSwitch, ccLLVM_Gcc, ccCLang, ccBcc, ccVcc,
-    ccTcc, ccEnv, ccIcl, ccIcc, ccClangCl
+    ccTcc, ccEnv, ccIcl, ccIcc, ccClangCl, ccHipcc, ccNvcc
 
   ExceptionSystem* = enum
     excNone,   # no exception system selected yet

compiler/pipelines.nim

@@ -13,7 +13,6 @@ when not defined(leanCompiler):
 import std/[syncio, objectdollar, assertions, tables, strutils, strtabs]
 import renderer
 import ic/replayer
-import nir/nir
 
 proc setPipeLinePass*(graph: ModuleGraph; pass: PipelinePass) =
   graph.pipelinePass = pass
@@ -45,10 +44,6 @@ proc processPipeline(graph: ModuleGraph; semNode: PNode; bModule: PPassContext):
       result = nil
   of EvalPass, InterpreterPass:
     result = interpreterCode(bModule, semNode)
-  of NirReplPass:
-    result = runCode(bModule, semNode)
-  of NirPass:
-    result = nirBackend(bModule, semNode)
   of NonePass:
     raiseAssert "use setPipeLinePass to set a proper PipelinePass"
 
@@ -111,8 +106,6 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
     case graph.pipelinePass
     of CgenPass:
       setupCgen(graph, module, idgen)
-    of NirPass:
-      openNirBackend(graph, module, idgen)
     of JSgenPass:
       when not defined(leanCompiler):
         setupJSgen(graph, module, idgen)
@@ -120,8 +113,6 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
         nil
     of EvalPass, InterpreterPass:
       setupEvalGen(graph, module, idgen)
-    of NirReplPass:
-      setupNirReplGen(graph, module, idgen)
     of GenDependPass:
       setupDependPass(graph, module, idgen)
     of Docgen2Pass:
@@ -209,10 +200,6 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
       discard finalJSCodeGen(graph, bModule, finalNode)
   of EvalPass, InterpreterPass:
     discard interpreterCode(bModule, finalNode)
-  of NirReplPass:
-    discard runCode(bModule, finalNode)
-  of NirPass:
-    closeNirBackend(bModule, finalNode)
   of SemPass, GenDependPass:
     discard
   of Docgen2Pass, Docgen2TexPass:

compiler/pragmas.nim

@@ -81,7 +81,7 @@ const
     wRequiresInit, wNoalias, wAlign, wNoInit} - {wExportNims, wNodecl} # why exclude these?
   varPragmas* = declPragmas + {wVolatile, wRegister, wThreadVar,
     wMagic, wHeader, wCompilerProc, wCore, wDynlib,
-    wNoInit, wCompileTime, wGlobal,
+    wNoInit, wCompileTime, wGlobal, wLiftLocals,
     wGensym, wInject, wCodegenDecl,
     wGuard, wGoto, wCursor, wNoalias, wAlign}
   constPragmas* = declPragmas + {wHeader, wMagic,
@@ -882,6 +882,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
 
     # number of pragmas increase/decrease with user pragma expansion
     inc c.instCounter
+    defer: dec c.instCounter
     if c.instCounter > 100:
       globalError(c.config, it.info, "recursive dependency: " & userPragma.name.s)
 
@@ -891,7 +892,6 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
     pragma(c, sym, userPragma.ast, validPragmas, isStatement)
     n.sons[i..i] = userPragma.ast.sons # expand user pragma with its content
     i.inc(userPragma.ast.len - 1) # inc by -1 is ok, user pragmas was empty
-    dec c.instCounter
   else:
     let k = whichKeyword(ident)
     if k in validPragmas:
@@ -1036,7 +1036,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         incl(sym.flags, sfGeneratedOp)
       of wNosinks:
         noVal(c, it)
-        incl(sym.flags, sfWasForwarded)
+        incl(sym.flags, sfNosinks)
       of wDynlib:
         processDynLib(c, it, sym)
       of wCompilerProc, wCore:
@@ -1308,7 +1308,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         noVal(c, it)
         if sym == nil: invalidPragma(c, it)
         else: sym.flags.incl sfUsed
-      of wLiftLocals: discard
+      of wLiftLocals:
+        sym.flags.incl(sfForceLift)
       of wRequires, wInvariant, wAssume, wAssert:
         pragmaProposition(c, it)
       of wEnsures:

compiler/sem.nim

@@ -222,66 +222,7 @@ proc shouldCheckCaseCovered(caseTyp: PType): bool =
   else:
     discard
 
-proc endsInNoReturn(n: PNode): bool =
-  ## check if expr ends the block like raising or call of noreturn procs do
-  result = false # assume it does return
-
-  template checkBranch(branch) =
-    if not endsInNoReturn(branch):
-      # proved a branch returns
-      return false
-
-  var it = n
-  # skip these beforehand, no special handling needed
-  while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0:
-    it = it.lastSon
-
-  case it.kind
-  of nkIfStmt:
-    var hasElse = false
-    for branch in it:
-      checkBranch:
-        if branch.len == 2:
-          branch[1]
-        elif branch.len == 1:
-          hasElse = true
-          branch[0]
-        else:
-          raiseAssert "Malformed `if` statement during endsInNoReturn"
-    # none of the branches returned
-    result = hasElse # Only truly a no-return when it's exhaustive
-  of nkCaseStmt:
-    let caseTyp = skipTypes(it[0].typ, abstractVar-{tyTypeDesc})
-    # semCase should already have checked for exhaustiveness in this case
-    # effectively the same as having an else
-    var hasElse = caseTyp.shouldCheckCaseCovered()
-
-    # actual noreturn checks
-    for i in 1 ..< it.len:
-      let branch = it[i]
-      checkBranch:
-        case branch.kind
-        of nkOfBranch:
-          branch[^1]
-        of nkElifBranch:
-          branch[1]
-        of nkElse:
-          hasElse = true
-          branch[0]
-        else:
-          raiseAssert "Malformed `case` statement in endsInNoReturn"
-    # Can only guarantee a noreturn if there is an else or it's exhaustive
-    result = hasElse
-  of nkTryStmt:
-    checkBranch(it[0])
-    for i in 1 ..< it.len:
-      let branch = it[i]
-      checkBranch(branch[^1])
-    # none of the branches returned
-    result = true
-  else:
-    result = it.kind in nkLastBlockStmts or
-      it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags
+proc endsInNoReturn(n: PNode): bool
 
 proc commonType*(c: PContext; x: PType, y: PNode): PType =
   # ignore exception raising branches in case/if expressions
@@ -313,6 +254,8 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     result.owner = getCurrOwner(c)
   else:
     result = newSym(kind, considerQuotedIdent(c, n), c.idgen, getCurrOwner(c), n.info)
+    if find(result.name.s, '`') >= 0:
+      result.flags.incl sfWasGenSym
   #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule:
   #  incl(result.flags, sfGlobal)
   when defined(nimsuggest):
@@ -322,7 +265,7 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
                  allowed: TSymFlags): PSym
   # identifier with visibility
 proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
-                        allowed: TSymFlags): PSym
+                        allowed: TSymFlags, fromTopLevel = false): PSym
 
 proc typeAllowedCheck(c: PContext; info: TLineInfo; typ: PType; kind: TSymKind;
                       flags: TTypeAllowedFlags = {}) =

compiler/semexprs.nim

@@ -562,7 +562,7 @@ proc semIs(c: PContext, n: PNode, flags: TExprFlags): PNode =
   result = isOpImpl(c, n, flags)
 
 proc semOpAux(c: PContext, n: PNode) =
-  const flags = {efDetermineType}
+  const flags = {efDetermineType, efAllowSymChoice}
   for i in 1..<n.len:
     var a = n[i]
     if a.kind == nkExprEqExpr and a.len == 2:
@@ -2944,6 +2944,18 @@ proc asBracketExpr(c: PContext; n: PNode): PNode =
           return result
   return nil
 
+proc isOpenArraySym(x: PNode): bool =
+  var x = x
+  while true:
+    case x.kind
+    of {nkAddr, nkHiddenAddr}:
+      x = x[0]
+    of {nkHiddenStdConv, nkHiddenDeref}:
+      x = x[1]
+    else:
+      break
+  result = x.kind == nkSym
+
 proc hoistParamsUsedInDefault(c: PContext, call, letSection, defExpr: var PNode) =
   # This takes care of complicated signatures such as:
   # proc foo(a: int, b = a)
@@ -2964,7 +2976,10 @@ proc hoistParamsUsedInDefault(c: PContext, call, letSection, defExpr: var PNode)
   if defExpr.kind == nkSym and defExpr.sym.kind == skParam and defExpr.sym.owner == call[0].sym:
     let paramPos = defExpr.sym.position + 1
 
-    if call[paramPos].skipAddr.kind != nkSym:
+    if call[paramPos].skipAddr.kind != nkSym and not (
+      skipTypes(call[paramPos].typ, abstractVar).kind in {tyOpenArray, tyVarargs} and
+      isOpenArraySym(call[paramPos])
+    ):
       let hoistedVarSym = newSym(skLet, getIdent(c.graph.cache, genPrefix), c.idgen,
                                  c.p.owner, letSection.info, c.p.owner.options)
       hoistedVarSym.typ = call[paramPos].typ

compiler/semfold.nim

@@ -701,10 +701,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
     except DivByZeroDefect:
       localError(g.config, n.info, "division by zero")
   of nkAddr:
-    var a = getConstExpr(m, n[0], idgen, g)
-    if a != nil:
-      result = n
-      n[0] = a
+    result = nil # don't fold paths containing nkAddr
   of nkBracket, nkCurly:
     result = copyNode(n)
     for son in n.items:
@@ -771,7 +768,8 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
   of nkCast:
     var a = getConstExpr(m, n[1], idgen, g)
     if a == nil: return
-    if n.typ != nil and n.typ.kind in NilableTypes:
+    if n.typ != nil and n.typ.kind in NilableTypes and
+        not (n.typ.kind == tyProc and a.typ.kind == tyProc):
       # we allow compile-time 'cast' for pointer types:
       result = a
       result.typ = n.typ

compiler/seminst.nim

@@ -56,8 +56,11 @@ iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TypeMapping): PS
       elif t.kind in {tyGenericParam, tyConcept}:
         localError(c.config, a.info, errCannotInstantiateX % q.name.s)
         t = errorType(c)
-      elif isUnresolvedStatic(t) and c.inGenericContext == 0 and
-          c.matchedConcept == nil:
+      elif isUnresolvedStatic(t) and (q.typ.kind == tyStatic or
+            (q.typ.kind == tyGenericParam and
+              q.typ.genericParamHasConstraints and
+              q.typ.genericConstraint.kind == tyStatic)) and
+          c.inGenericContext == 0 and c.matchedConcept == nil:
         # generic/concept type bodies will try to instantiate static values but
         # won't actually use them
         localError(c.config, a.info, errCannotInstantiateX % q.name.s)
@@ -342,7 +345,8 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
   # generates an instantiated proc
   if c.instCounter > 50:
     globalError(c.config, info, "generic instantiation too nested")
-  inc(c.instCounter)
+  inc c.instCounter
+  defer: dec c.instCounter
   # careful! we copy the whole AST including the possibly nil body!
   var n = copyTree(fn.ast)
   # NOTE: for access of private fields within generics from a different module
@@ -396,6 +400,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
   for _, param in paramTypes(result.typ):
     entry.concreteTypes[i] = param
     inc i
+  #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " ", entry.concreteTypes.len
   if tfTriggersCompileTime in result.typ.flags:
     incl(result.flags, sfCompileTime)
   n[genericParamsPos] = c.graph.emptyNode
@@ -424,7 +429,9 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
     if result.magic notin {mSlice, mTypeOf}:
       # 'toOpenArray' is special and it is allowed to return 'openArray':
       paramsTypeCheck(c, result.typ)
+    #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " <-- NEW PROC!", " ", entry.concreteTypes.len
   else:
+    #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " <-- CACHED! ", typeToString(oldPrc.typ), " ", entry.concreteTypes.len
     result = oldPrc
   popProcCon(c)
   popInfoContext(c.config)
@@ -433,7 +440,6 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
   popOwner(c)
   c.currentScope = oldScope
   discard c.friendModules.pop()
-  dec(c.instCounter)
   c.matchedConcept = oldMatchedConcept
   if result.kind == skMethod: finishMethod(c, result)
 

compiler/semstmts.nim

@@ -132,17 +132,140 @@ proc semExprBranchScope(c: PContext, n: PNode; expectedType: PType = nil): PNode
   closeScope(c)
 
 const
-  skipForDiscardable = {nkIfStmt, nkIfExpr, nkCaseStmt, nkOfBranch,
-    nkElse, nkStmtListExpr, nkTryStmt, nkFinally, nkExceptBranch,
+  skipForDiscardable = {nkStmtList, nkStmtListExpr,
+    nkOfBranch, nkElse, nkFinally, nkExceptBranch,
     nkElifBranch, nkElifExpr, nkElseExpr, nkBlockStmt, nkBlockExpr,
     nkHiddenStdConv, nkHiddenDeref}
 
 proc implicitlyDiscardable(n: PNode): bool =
-  var n = n
-  while n.kind in skipForDiscardable: n = n.lastSon
-  result = n.kind in nkLastBlockStmts or
-           (isCallExpr(n) and n[0].kind == nkSym and
-           sfDiscardable in n[0].sym.flags)
+  # same traversal as endsInNoReturn
+  template checkBranch(branch) =
+    if not implicitlyDiscardable(branch):
+      return false
+
+  var it = n
+  # skip these beforehand, no special handling needed
+  while it.kind in skipForDiscardable and it.len > 0:
+    it = it.lastSon
+
+  case it.kind
+  of nkIfExpr, nkIfStmt:
+    for branch in it:
+      checkBranch:
+        if branch.len == 2:
+          branch[1]
+        elif branch.len == 1:
+          branch[0]
+        else:
+          raiseAssert "Malformed `if` statement during implicitlyDiscardable"
+    # all branches are discardable
+    result = true
+  of nkCaseStmt:
+    for i in 1 ..< it.len:
+      let branch = it[i]
+      checkBranch:
+        case branch.kind
+        of nkOfBranch:
+          branch[^1]
+        of nkElifBranch:
+          branch[1]
+        of nkElse:
+          branch[0]
+        else:
+          raiseAssert "Malformed `case` statement in endsInNoReturn"
+    # all branches are discardable
+    result = true
+  of nkTryStmt:
+    checkBranch(it[0])
+    for i in 1 ..< it.len:
+      let branch = it[i]
+      if branch.kind != nkFinally:
+        checkBranch(branch[^1])
+    # all branches are discardable
+    result = true
+  of nkCallKinds:
+    result = it[0].kind == nkSym and {sfDiscardable, sfNoReturn} * it[0].sym.flags != {}
+  of nkLastBlockStmts:
+    result = true
+  else:
+    result = false
+
+proc endsInNoReturn(n: PNode, returningNode: var PNode): bool =
+  ## check if expr ends the block like raising or call of noreturn procs do
+  result = false # assume it does return
+
+  template checkBranch(branch) =
+    if not endsInNoReturn(branch, returningNode):
+      # proved a branch returns
+      return false
+
+  var it = n
+  # skip these beforehand, no special handling needed
+  while it.kind in skipForDiscardable and it.len > 0:
+    it = it.lastSon
+
+  case it.kind
+  of nkIfExpr, nkIfStmt:
+    var hasElse = false
+    for branch in it:
+      checkBranch:
+        if branch.len == 2:
+          branch[1]
+        elif branch.len == 1:
+          hasElse = true
+          branch[0]
+        else:
+          raiseAssert "Malformed `if` statement during endsInNoReturn"
+    # none of the branches returned
+    result = hasElse # Only truly a no-return when it's exhaustive
+  of nkCaseStmt:
+    let caseTyp = skipTypes(it[0].typ, abstractVar-{tyTypeDesc})
+    # semCase should already have checked for exhaustiveness in this case
+    # effectively the same as having an else
+    var hasElse = caseTyp.shouldCheckCaseCovered()
+
+    # actual noreturn checks
+    for i in 1 ..< it.len:
+      let branch = it[i]
+      checkBranch:
+        case branch.kind
+        of nkOfBranch:
+          branch[^1]
+        of nkElifBranch:
+          branch[1]
+        of nkElse:
+          hasElse = true
+          branch[0]
+        else:
+          raiseAssert "Malformed `case` statement in endsInNoReturn"
+    # Can only guarantee a noreturn if there is an else or it's exhaustive
+    result = hasElse
+  of nkTryStmt:
+    checkBranch(it[0])
+    var lastIndex = it.len - 1
+    if it[lastIndex].kind == nkFinally:
+      # if finally is noreturn, then the entire statement is noreturn
+      if endsInNoReturn(it[lastIndex][^1], returningNode):
+        return true
+      dec lastIndex
+    for i in 1 .. lastIndex:
+      let branch = it[i]
+      checkBranch(branch[^1])
+    # none of the branches returned
+    result = true
+  of nkLastBlockStmts:
+    result = true
+  of nkCallKinds:
+    result = it[0].kind == nkSym and sfNoReturn in it[0].sym.flags
+    if not result:
+      returningNode = it
+  else:
+    result = false
+    returningNode = it
+
+proc endsInNoReturn(n: PNode): bool =
+  var dummy: PNode = nil
+  result = endsInNoReturn(n, dummy)
 
 proc fixNilType(c: PContext; n: PNode) =
   if isAtom(n):
@@ -160,18 +283,15 @@ proc discardCheck(c: PContext, result: PNode, flags: TExprFlags) =
     if implicitlyDiscardable(result):
       var n = newNodeI(nkDiscardStmt, result.info, 1)
       n[0] = result
+      # notes that it doesn't transform nodes into discard statements
     elif result.typ.kind != tyError and c.config.cmd != cmdInteractive:
       if result.typ.kind == tyNone:
         localError(c.config, result.info, "expression has no type: " &
                renderTree(result, {renderNoComments}))
       else:
-        var n = result
-        while n.kind in skipForDiscardable:
-          if n.kind == nkTryStmt: n = n[0]
-          else: n = n.lastSon
-
         # Ignore noreturn procs since they don't have a type
-        if n.endsInNoReturn:
+        var n = result
+        if result.endsInNoReturn(n):
           return
 
         var s = "expression '" & $n & "' is of type '" &
@@ -361,7 +481,7 @@ proc identWithin(n: PNode, s: PIdent): bool =
 
 proc semIdentDef(c: PContext, n: PNode, kind: TSymKind, reportToNimsuggest = true): PSym =
   if isTopLevel(c):
-    result = semIdentWithPragma(c, kind, n, {sfExported})
+    result = semIdentWithPragma(c, kind, n, {sfExported}, fromTopLevel = true)
     incl(result.flags, sfGlobal)
     #if kind in {skVar, skLet}:
     #  echo "global variable here ", n.info, " ", result.name.s
@@ -1655,8 +1775,13 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
               assignType(s.typ, t)
               s.typ.itemId = t.itemId     # same id
         var hasError = false
-        if s.typ.kind in {tyObject, tyTuple} and not s.typ.n.isNil:
-          checkForMetaFields(c, s.typ.n, hasError)
+        let baseType = s.typ.safeSkipTypes(abstractPtrs)
+        if baseType.kind in {tyObject, tyTuple} and not baseType.n.isNil and
+          (x.kind in {nkObjectTy, nkTupleTy} or
+           (x.kind in {nkRefTy, nkPtrTy} and x.len == 1 and
+           x[0].kind in {nkObjectTy, nkTupleTy})
+          ):
+          checkForMetaFields(c, baseType.n, hasError)
         if not hasError:
           checkConstructedType(c.config, s.info, s.typ)
 

compiler/semtypes.nim

@@ -540,7 +540,7 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
     result = newSymG(kind, n, c)
 
 proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
-                        allowed: TSymFlags): PSym =
+                        allowed: TSymFlags, fromTopLevel = false): PSym =
   if n.kind == nkPragmaExpr:
     checkSonsLen(n, 2, c.config)
     result = semIdentVis(c, kind, n[0], allowed)
@@ -555,11 +555,15 @@ proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
     else: discard
   else:
     result = semIdentVis(c, kind, n, allowed)
+    let invalidPragmasForPush = if fromTopLevel and sfWasGenSym notin result.flags:
+      {}
+    else:
+      {wExportc, wExportCpp, wDynlib}
     case kind
     of skField: implicitPragmas(c, result, n.info, fieldPragmas)
-    of skVar:   implicitPragmas(c, result, n.info, varPragmas)
-    of skLet:   implicitPragmas(c, result, n.info, letPragmas)
-    of skConst: implicitPragmas(c, result, n.info, constPragmas)
+    of skVar:   implicitPragmas(c, result, n.info, varPragmas-invalidPragmasForPush)
+    of skLet:   implicitPragmas(c, result, n.info, letPragmas-invalidPragmasForPush)
+    of skConst: implicitPragmas(c, result, n.info, constPragmas-invalidPragmasForPush)
     else: discard
 
 proc checkForOverlap(c: PContext, t: PNode, currentEx, branchIndex: int) =

compiler/sigmatch.nim

@@ -1260,13 +1260,14 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
     subtypeCheck()
   of tyArray:
     a = reduceToBase(a)
-    case a.kind
-    of tyArray:
+    if a.kind == tyArray:
       var fRange = f.indexType
       var aRange = a.indexType
       if fRange.kind in {tyGenericParam, tyAnything}:
         var prev = idTableGet(c.bindings, fRange)
         if prev == nil:
+          if typeRel(c, fRange, aRange) == isNone:
+            return isNone
           put(c, fRange, a.indexType)
           fRange = a
         else:
@@ -1279,7 +1280,6 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
         result = isGeneric
       else:
         result = typeRel(c, ff, aa, flags)
-
       if result < isGeneric:
         if nimEnableCovariance and
            trNoCovariance notin flags and
@@ -1290,6 +1290,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
           return isNone
 
       if fRange.rangeHasUnresolvedStatic:
+        if aRange.kind in {tyGenericParam} and aRange.reduceToBase() == aRange:
+          return
         return inferStaticsInRange(c, fRange, a)
       elif c.c.matchedConcept != nil and aRange.rangeHasUnresolvedStatic:
         return inferStaticsInRange(c, aRange, f)
@@ -1298,7 +1300,6 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
       else:
         if lengthOrd(c.c.config, fRange) != lengthOrd(c.c.config, aRange):
           result = isNone
-    else: discard
   of tyUncheckedArray:
     if a.kind == tyUncheckedArray:
       result = typeRel(c, elementType(f), elementType(a), flags)
@@ -2491,7 +2492,7 @@ proc arrayConstr(c: PContext, n: PNode): PType =
   result = newTypeS(tyArray, c)
   rawAddSon(result, makeRangeType(c, 0, 0, n.info))
   addSonSkipIntLit(result, skipTypes(n.typ,
-      {tyGenericInst, tyVar, tyLent, tyOrdinal}), c.idgen)
+      {tyVar, tyLent, tyOrdinal}), c.idgen)
 
 proc arrayConstr(c: PContext, info: TLineInfo): PType =
   result = newTypeS(tyArray, c)

compiler/sinkparameter_inference.nim

@@ -28,7 +28,7 @@ proc checkForSink*(config: ConfigRef; idgen: IdGenerator; owner: PSym; arg: PNod
         arg.sym.typ.kind notin {tyVar, tySink, tyOwned}:
       # Watch out: cannot do this inference for procs with forward
       # declarations.
-      if sfWasForwarded notin owner.flags:
+      if {sfWasForwarded, sfNosinks} * owner.flags == {}:
         let argType = arg.sym.typ
 
         let sinkType = newType(tySink, idgen, owner)
@@ -43,9 +43,9 @@ proc checkForSink*(config: ConfigRef; idgen: IdGenerator; owner: PSym; arg: PNod
         #message(config, arg.info, warnUser,
         #  ("turned '$1' to a sink parameter") % [$arg])
         #echo config $ arg.info, " turned into a sink parameter ", arg.sym.name.s
-      elif sfWasForwarded notin arg.sym.flags:
+      elif {sfWasForwarded, sfNosinks} * arg.sym.flags == {}:
         # we only report every potential 'sink' parameter only once:
-        incl arg.sym.flags, sfWasForwarded
+        incl arg.sym.flags, sfNosinks
         message(config, arg.info, hintPerformance,
           "could not turn '$1' to a sink parameter" % [arg.sym.name.s])
       #echo config $ arg.info, " candidate for a sink parameter here"

compiler/spawn.nim

@@ -109,6 +109,16 @@ stmtList:
 
 """
 
+proc castToVoidPointer(g: ModuleGraph, n: PNode, fvField: PNode): PNode =
+  if g.config.backend == backendCpp:
+    result = fvField
+  else:
+    let ptrType = getSysType(g, n.info, tyPointer)
+    result = newNodeI(nkCast, fvField.info)
+    result.add newNodeI(nkEmpty, fvField.info)
+    result.add fvField
+    result.typ = ptrType
+
 proc createWrapperProc(g: ModuleGraph; f: PNode; threadParam, argsParam: PSym;
                        varSection, varInit, call, barrier, fv: PNode;
                        idgen: IdGenerator;
@@ -156,8 +166,9 @@ proc createWrapperProc(g: ModuleGraph; f: PNode; threadParam, argsParam: PSym;
     if barrier == nil:
       # by now 'fv' is shared and thus might have beeen overwritten! we need
       # to use the thread-local view instead:
+      let castExpr = castToVoidPointer(g, f, threadLocalProm.newSymNode)
       body.add callCodegenProc(g, "nimFlowVarSignal", threadLocalProm.info,
-        threadLocalProm.newSymNode)
+        castExpr)
   else:
     body.add call
   if barrier != nil:
@@ -413,7 +424,8 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
     # create flowVar:
     result.add newFastAsgnStmt(fvField, callProc(spawnExpr[^1]))
     if barrier == nil:
-      result.add callCodegenProc(g, "nimFlowVarCreateSemaphore", fvField.info, fvField)
+      let castExpr = castToVoidPointer(g, n, fvField)
+      result.add callCodegenProc(g, "nimFlowVarCreateSemaphore", fvField.info, castExpr)
 
   elif spawnKind == srByVar:
     var field = newSym(skField, getIdent(g.cache, "fv"), idgen, owner, n.info, g.config.options)

compiler/transf.nim

@@ -56,6 +56,7 @@ type
     contSyms, breakSyms: seq[PSym]  # to transform 'continue' and 'break'
     deferDetected, tooEarly: bool
     isIntroducingNewLocalVars: bool  # true if we are in `introducingNewLocalVars` (don't transform yields)
+    inAddr: bool
     flags: TransformFlags
     graph: ModuleGraph
     idgen: IdGenerator
@@ -97,10 +98,7 @@ proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode =
   r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias, tySink})
   incl(r.flags, sfFromGeneric)
   let owner = getCurrOwner(c)
-  if owner.isIterator and not c.tooEarly:
-    result = freshVarForClosureIter(c.graph, r, c.idgen, owner)
-  else:
-    result = newSymNode(r)
+  result = newSymNode(r)
 
 proc transform(c: PTransf, n: PNode): PNode
 
@@ -176,13 +174,10 @@ proc transformSym(c: PTransf, n: PNode): PNode =
 
 proc freshVar(c: PTransf; v: PSym): PNode =
   let owner = getCurrOwner(c)
-  if owner.isIterator and not c.tooEarly:
-    result = freshVarForClosureIter(c.graph, v, c.idgen, owner)
-  else:
-    var newVar = copySym(v, c.idgen)
-    incl(newVar.flags, sfFromGeneric)
-    newVar.owner = owner
-    result = newSymNode(newVar)
+  var newVar = copySym(v, c.idgen)
+  incl(newVar.flags, sfFromGeneric)
+  newVar.owner = owner
+  result = newSymNode(newVar)
 
 proc transformVarSection(c: PTransf, v: PNode): PNode =
   result = newTransNode(v)
@@ -513,6 +508,7 @@ proc generateThunk(c: PTransf; prc: PNode, dest: PType): PNode =
 
   # we cannot generate a proper thunk here for GC-safety reasons
   # (see internal documentation):
+  if jsNoLambdaLifting in c.graph.config.legacyFeatures and c.graph.config.backend == backendJs: return prc
   result = newNodeIT(nkClosure, prc.info, dest)
   var conv = newNodeIT(nkHiddenSubConv, prc.info, dest)
   conv.add(newNodeI(nkEmpty, prc.info))
@@ -1061,7 +1057,10 @@ proc transform(c: PTransf, n: PNode): PNode =
   of nkHiddenAddr:
     result = transformAddrDeref(c, n, {nkHiddenDeref})
   of nkAddr:
+    let oldInAddr = c.inAddr
+    c.inAddr = true
     result = transformAddrDeref(c, n, {nkDerefExpr, nkHiddenDeref})
+    c.inAddr = oldInAddr
   of nkDerefExpr:
     result = transformAddrDeref(c, n, {nkAddr, nkHiddenAddr})
   of nkHiddenDeref:
@@ -1141,7 +1140,7 @@ proc transform(c: PTransf, n: PNode): PNode =
   let exprIsPointerCast = n.kind in {nkCast, nkConv, nkHiddenStdConv} and
                           n.typ != nil and
                           n.typ.kind == tyPointer
-  if not exprIsPointerCast:
+  if not exprIsPointerCast and not c.inAddr:
     var cnst = getConstExpr(c.module, result, c.idgen, c.graph)
     # we inline constants if they are not complex constants:
     if cnst != nil and not dontInlineConstant(n, cnst):

compiler/types.nim

@@ -115,7 +115,7 @@ proc isPureObject*(typ: PType): bool =
 proc isUnsigned*(t: PType): bool =
   t.skipTypes(abstractInst).kind in {tyChar, tyUInt..tyUInt64}
 
-proc getOrdValue*(n: PNode; onError = high(Int128)): Int128 =
+proc getOrdValueAux*(n: PNode, err: var bool): Int128 =
   var k = n.kind
   if n.typ != nil and n.typ.skipTypes(abstractInst).kind in {tyChar, tyUInt..tyUInt64}:
     k = nkUIntLit
@@ -131,13 +131,22 @@ proc getOrdValue*(n: PNode; onError = high(Int128)): Int128 =
     toInt128(n.intVal)
   of nkNilLit:
     int128.Zero
-  of nkHiddenStdConv: getOrdValue(n[1], onError)
+  of nkHiddenStdConv:
+    getOrdValueAux(n[1], err)
   else:
-    # XXX: The idea behind the introduction of int128 was to finally
-    # have all calculations numerically far away from any
-    # overflows. This command just introduces such overflows and
-    # should therefore really be revisited.
-    onError
+    err = true
+    int128.Zero
+
+proc getOrdValue*(n: PNode): Int128 =
+  var err: bool = false
+  result = getOrdValueAux(n, err)
+  #assert err == false
+
+proc getOrdValue*(n: PNode, onError: Int128): Int128 =
+  var err = false
+  result = getOrdValueAux(n, err)
+  if err:
+    result = onError
 
 proc getFloatValue*(n: PNode): BiggestFloat =
   case n.kind
@@ -1316,9 +1325,17 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     result = sameTypeOrNilAux(a.elementType, b.elementType, c) and
         sameValue(a.n[0], b.n[0]) and
         sameValue(a.n[1], b.n[1])
-  of tyGenericInst, tyAlias, tyInferred, tyIterable:
+  of tyAlias, tyInferred, tyIterable:
     cycleCheck()
     result = sameTypeAux(a.skipModifier, b.skipModifier, c)
+  of tyGenericInst:
+    # BUG #23445
+    # The type system must distinguish between `T[int] = object #[empty]#`
+    # and `T[float] = object #[empty]#`!
+    cycleCheck()
+    for ff, aa in underspecifiedPairs(a, b, 1, -1):
+      if not sameTypeAux(ff, aa, c): return false
+    result = sameTypeAux(a.skipModifier, b.skipModifier, c)
   of tyNone: result = false
   of tyConcept:
     result = exprStructuralEquivalent(a.n, b.n)

compiler/vmgen.nim

@@ -1559,7 +1559,8 @@ proc checkCanEval(c: PCtx; n: PNode) =
     elif s.kind == skParam and s.typ.kind == tyTypeDesc: discard
     else: cannotEval(c, n)
   elif s.kind in {skProc, skFunc, skConverter, skMethod,
-                  skIterator} and sfForward in s.flags:
+                  skIterator} and sfWasForwarded in s.flags and
+                  s.originatingModule == c.module: # forbides recursive calls from the same module
     cannotEval(c, n)
 
 template needsAdditionalCopy(n): untyped =

config/nimdoc.cfg

@@ -232,6 +232,10 @@ doc.file = """<?xml version="1.0" encoding="utf-8" ?>
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>$title</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -255,9 +259,6 @@ doc.file = """<?xml version="1.0" encoding="utf-8" ?>
     </div>
   </div>
   $analytics
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>
 """

doc/manual.md

@@ -8703,7 +8703,7 @@ after the last specified parameter. Nim string values will be converted to C
 strings automatically:
 
   ```Nim
-  proc printf(formatstr: cstring) {.nodecl, varargs.}
+  proc printf(formatstr: cstring) {.header: "<stdio.h>", varargs.}
 
   printf("hallo %s", "world") # "world" will be passed as C string
   ```

doc/nimc.md

@@ -481,6 +481,28 @@ They are:
 5. nl_types. No headers for this.
 6. As mmap is not supported, the nimAllocPagesViaMalloc option has to be used.
 
+GPU Compilation
+===============
+
+Compiling for GPU computation can be achieved with `--cc:nvcc` for CUDA with nvcc, or with `--cc:hipcc` for AMD GPUs with HIP. Both compilers require building for C++ with `nim cpp`.
+
+Here's a very simple CUDA kernel example using emit, which can be compiled with `nim cpp --cc:nvcc --define:"useMalloc" hello_kernel.nim` assuming you have the CUDA toolkit installed.
+
+```nim
+{.emit: """
+__global__ void add(int a, int b) {
+  int c;
+  c = a + b;
+}
+""".}
+
+proc main() =
+  {.emit: """
+  add<<<1,1>>>(2,7);
+  """.}
+
+main()
+```
 
 DLL generation
 ==============

koch.nim

@@ -11,7 +11,7 @@
 
 const
   # examples of possible values for repos: Head, ea82b54
-  NimbleStableCommit = "f8bd7b5fa6ea7a583b411b5959b06e6b5eb23667" # master
+  NimbleStableCommit = "3b119f6fa135429a704f9c60f0b8944906398eea" # master
   AtlasStableCommit = "5faec3e9a33afe99a7d22377dd1b45a5391f5504"
   ChecksumsStableCommit = "025bcca3915a1b9f19878cea12ad68f9884648fc"
   SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01"

lib/core/macros.nim

@@ -277,7 +277,7 @@ when (NimMajor, NimMinor, NimPatch) >= (1, 3, 5) or defined(nimSymImplTransform)
     ## note that code transformations are implementation dependent and subject to change.
     ## See an example in `tests/macros/tmacros_various.nim`.
 
-proc owner*(sym: NimNode): NimNode {.magic: "SymOwner", noSideEffect.}
+proc owner*(sym: NimNode): NimNode {.magic: "SymOwner", noSideEffect, deprecated.}
   ## Accepts a node of kind `nnkSym` and returns its owner's symbol.
   ## The meaning of 'owner' depends on `sym`'s `NimSymKind` and declaration
   ## context. For top level declarations this is an `nskModule` symbol,

lib/nimbase.h

@@ -272,11 +272,15 @@ __EMSCRIPTEN__
 #elif defined(__cplusplus)
 #define NIM_STATIC_ASSERT(x, msg) static_assert((x), msg)
 #else
-#define NIM_STATIC_ASSERT(x, msg) typedef int NIM_STATIC_ASSERT_AUX[(x) ? 1 : -1];
+#define _NIM_STATIC_ASSERT_FINAL(x, append_name) typedef int NIM_STATIC_ASSERT_AUX ## append_name[(x) ? 1 : -1];
+#define _NIM_STATIC_ASSERT_STAGE_3(x, line)      _NIM_STATIC_ASSERT_FINAL(x, _AT_LINE_##line)
+#define _NIM_STATIC_ASSERT_STAGE_2(x, line)      _NIM_STATIC_ASSERT_STAGE_3(x, line)
+#define NIM_STATIC_ASSERT(x, msg)                _NIM_STATIC_ASSERT_STAGE_2(x,__LINE__)
 // On failure, your C compiler will say something like:
-//   "error: 'NIM_STATIC_ASSERT_AUX' declared as an array with a negative size"
-// we could use a better fallback to also show line number, using:
-// http://www.pixelbeat.org/programming/gcc/static_assert.html
+//   "error: 'NIM_STATIC_ASSERT_AUX_AT_LINE_XXX' declared as an array with a negative size"
+// Adding the line number helps to avoid redefinitions which are not allowed in
+// old GCC versions, however the order of evaluation for __LINE__ is a little tricky,
+// hence all the helper macros. See https://stackoverflow.com/a/3385694 for more info.
 #endif
 
 /* C99 compiler? */

lib/pure/fenv.nim

@@ -12,7 +12,7 @@
 ## The types, vars and procs are bindings for the C standard library
 ## [<fenv.h>](https://en.cppreference.com/w/c/numeric/fenv) header.
 
-when defined(posix) and not defined(genode):
+when defined(posix) and not defined(genode) and not defined(macosx):
   {.passl: "-lm".}
 
 var

lib/pure/hashes.nim

@@ -379,6 +379,158 @@ proc hashVmImplChar(x: openArray[char], sPos, ePos: int): Hash =
 proc hashVmImplByte(x: openArray[byte], sPos, ePos: int): Hash =
   raiseAssert "implementation override in compiler/vmops.nim"
 
+const k0 = 0xc3a5c85c97cb3127u64 # Primes on (2^63, 2^64) for various uses
+const k1 = 0xb492b66fbe98f273u64
+const k2 = 0x9ae16a3b2f90404fu64
+
+proc load4e(s: openArray[byte], o=0): uint32 {.inline.} =
+  uint32(s[o + 3]) shl 24 or uint32(s[o + 2]) shl 16 or
+  uint32(s[o + 1]) shl  8 or uint32(s[o + 0])
+
+proc load8e(s: openArray[byte], o=0): uint64 {.inline.} =
+  uint64(s[o + 7]) shl 56 or uint64(s[o + 6]) shl 48 or
+  uint64(s[o + 5]) shl 40 or uint64(s[o + 4]) shl 32 or
+  uint64(s[o + 3]) shl 24 or uint64(s[o + 2]) shl 16 or
+  uint64(s[o + 1]) shl  8 or uint64(s[o + 0])
+
+proc load4(s: openArray[byte], o=0): uint32 {.inline.} =
+  when nimvm: result = load4e(s, o)
+  else:
+    when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof
+    else: result = load4e(s, o)
+
+proc load8(s: openArray[byte], o=0): uint64 {.inline.} =
+  when nimvm: result = load8e(s, o)
+  else:
+    when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof
+    else: result = load8e(s, o)
+
+proc lenU(s: openArray[byte]): uint64 {.inline.} = s.len.uint64
+
+proc shiftMix(v: uint64): uint64 {.inline.} = v xor (v shr 47)
+
+proc rotR(v: uint64; bits: cint): uint64 {.inline.} =
+  (v shr bits) or (v shl (64 - bits))
+
+proc len16(u: uint64; v: uint64; mul: uint64): uint64 {.inline.} =
+  var a = (u xor v)*mul
+  a = a xor (a shr 47)
+  var b = (v xor a)*mul
+  b = b xor (b shr 47)
+  b*mul
+
+proc len0_16(s: openArray[byte]): uint64 {.inline.} =
+  if s.len >= 8:
+    let mul = k2 + 2*s.lenU
+    let a   = load8(s) + k2
+    let b   = load8(s, s.len - 8)
+    let c   = rotR(b, 37)*mul + a
+    let d   = (rotR(a, 25) + b)*mul
+    len16 c, d, mul
+  elif s.len >= 4:
+    let mul = k2 + 2*s.lenU
+    let a   = load4(s).uint64
+    len16 s.lenU + (a shl 3), load4(s, s.len - 4), mul
+  elif s.len > 0:
+    let a = uint32(s[0])
+    let b = uint32(s[s.len shr 1])
+    let c = uint32(s[s.len - 1])
+    let y = a      + (b shl 8)
+    let z = s.lenU + (c shl 2)
+    shiftMix(y*k2 xor z*k0)*k2
+  else: k2      # s.len == 0
+
+proc len17_32(s: openArray[byte]): uint64 {.inline.} =
+  let mul = k2 + 2*s.lenU
+  let a = load8(s)*k1
+  let b = load8(s, 8)
+  let c = load8(s, s.len - 8)*mul
+  let d = load8(s, s.len - 16)*k2
+  len16 rotR(a + b, 43) + rotR(c, 30) + d, a + rotR(b + k2, 18) + c, mul
+
+proc len33_64(s: openArray[byte]): uint64 {.inline.} =
+  let mul = k2 + 2*s.lenU
+  let a = load8(s)*k2
+  let b = load8(s, 8)
+  let c = load8(s, s.len - 8)*mul
+  let d = load8(s, s.len - 16)*k2
+  let y = rotR(a + b, 43) + rotR(c, 30) + d
+  let z = len16(y, a + rotR(b + k2, 18) + c, mul)
+  let e = load8(s, 16)*mul
+  let f = load8(s, 24)
+  let g = (y + load8(s, s.len - 32))*mul
+  let h = (z + load8(s, s.len - 24))*mul
+  len16 rotR(e + f, 43) + rotR(g, 30) + h, e + rotR(f + a, 18) + g, mul
+
+type Pair = tuple[first, second: uint64]
+
+proc weakLen32withSeeds2(w, x, y, z, a, b: uint64): Pair {.inline.} =
+  var a = a + w
+  var b = rotR(b + a + z, 21)
+  let c = a
+  a += x
+  a += y
+  b += rotR(a, 44)
+  result[0] = a + z
+  result[1] = b + c
+
+proc weakLen32withSeeds(s: openArray[byte]; o: int; a,b: uint64): Pair {.inline.} =
+  weakLen32withSeeds2 load8(s, o     ), load8(s, o + 8),
+                      load8(s, o + 16), load8(s, o + 24), a, b
+
+proc hashFarm(s: openArray[byte]): uint64 {.inline.} =
+  if s.len <= 16: return len0_16(s)
+  if s.len <= 32: return len17_32(s)
+  if s.len <= 64: return len33_64(s)
+  const seed = 81u64 # not const to use input `h`
+  var
+    o = 0         # s[] ptr arith -> variable origin variable `o`
+    x = seed
+    y = seed*k1 + 113
+    z = shiftMix(y*k2 + 113)*k2
+    v, w: Pair
+  x = x*k2 + load8(s)
+  let eos    = ((s.len - 1) div 64)*64
+  let last64 = eos + ((s.len - 1) and 63) - 63
+  while true:
+    x = rotR(x + y + v[0] + load8(s, o+8), 37)*k1
+    y = rotR(y + v[1] + load8(s, o+48), 42)*k1
+    x = x xor w[1]
+    y += v[0] + load8(s, o+40)
+    z = rotR(z + w[0], 33)*k1
+    v = weakLen32withSeeds(s, o+0 , v[1]*k1, x + w[0])
+    w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16))
+    swap z, x
+    inc o, 64
+    if o == eos: break
+  let mul = k1 + ((z and 0xff) shl 1)
+  o = last64
+  w[0] += (s.lenU - 1) and 63
+  v[0] += w[0]
+  w[0] += v[0]
+  x = rotR(x + y + v[0] + load8(s, o+8), 37)*mul
+  y = rotR(y + v[1] + load8(s, o+48), 42)*mul
+  x = x xor w[1]*9
+  y += v[0]*9 + load8(s, o+40)
+  z = rotR(z + w[0], 33)*mul
+  v = weakLen32withSeeds(s, o+0 , v[1]*mul, x + w[0])
+  w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16))
+  swap z, x
+  len16 len16(v[0],w[0],mul) + shiftMix(y)*k0 + z, len16(v[1],w[1],mul) + x, mul
+
+template jsNoInt64: untyped =
+  when defined js:
+    when compiles(compileOption("jsbigint64")):
+      when not compileOption("jsbigint64"): true
+      else: false
+    else: false
+  else: false
+const sHash2 = (when defined(nimStringHash2) or jsNoInt64(): true else: false)
+
+template maybeFailJS_Number =
+  when jsNoInt64() and not defined(nimStringHash2):
+    {.error: "Must use `-d:nimStringHash2` when using `--jsbigint64:off`".}
+
 proc hash*(x: string): Hash =
   ## Efficient hashing of strings.
   ##
@@ -387,11 +539,14 @@ proc hash*(x: string): Hash =
   ## * `hashIgnoreCase <#hashIgnoreCase,string>`_
   runnableExamples:
     doAssert hash("abracadabra") != hash("AbracadabrA")
-
-  when nimvm:
-    result = hashVmImpl(x, 0, high(x))
+  maybeFailJS_Number()
+  when not sHash2:
+    result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high)))
   else:
-    result = murmurHash(toOpenArrayByte(x, 0, high(x)))
+    #when nimvm:
+    #  result = hashVmImpl(x, 0, high(x))
+    when true:
+      result = murmurHash(toOpenArrayByte(x, 0, high(x)))
 
 proc hash*(x: cstring): Hash =
   ## Efficient hashing of null-terminated strings.
@@ -400,14 +555,22 @@ proc hash*(x: cstring): Hash =
     doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA")
     doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA")
 
-  when nimvm:
-    hashVmImpl(x, 0, high(x))
-  else:
-    when not defined(js):
-      murmurHash(toOpenArrayByte(x, 0, x.high))
-    else:
+  maybeFailJS_Number()
+  when not sHash2:
+    when defined js:
       let xx = $x
-      murmurHash(toOpenArrayByte(xx, 0, high(xx)))
+      result = cast[Hash](hashFarm(toOpenArrayByte(xx, 0, xx.high)))
+    else:
+      result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high)))
+  else:
+    #when nimvm:
+    #  result = hashVmImpl(x, 0, high(x))
+    when true:
+      when not defined(js):
+        result = murmurHash(toOpenArrayByte(x, 0, x.high))
+      else:
+        let xx = $x
+        result = murmurHash(toOpenArrayByte(xx, 0, high(xx)))
 
 proc hash*(sBuf: string, sPos, ePos: int): Hash =
   ## Efficient hashing of a string buffer, from starting
@@ -418,7 +581,11 @@ proc hash*(sBuf: string, sPos, ePos: int): Hash =
     var a = "abracadabra"
     doAssert hash(a, 0, 3) == hash(a, 7, 10)
 
-  murmurHash(toOpenArrayByte(sBuf, sPos, ePos))
+  maybeFailJS_Number()
+  when not sHash2:
+    result = cast[Hash](hashFarm(toOpenArrayByte(sBuf, sPos, ePos)))
+  else:
+    murmurHash(toOpenArrayByte(sBuf, sPos, ePos))
 
 proc hashIgnoreStyle*(x: string): Hash =
   ## Efficient hashing of strings; style is ignored.
@@ -553,12 +720,18 @@ proc hash*[A](x: openArray[A]): Hash =
   ## Efficient hashing of arrays and sequences.
   ## There must be a `hash` proc defined for the element type `A`.
   when A is byte:
-    result = murmurHash(x)
-  elif A is char:
-    when nimvm:
-      result = hashVmImplChar(x, 0, x.high)
+    when not sHash2:
+      result = cast[Hash](hashFarm(x))
     else:
-      result = murmurHash(toOpenArrayByte(x, 0, x.high))
+      result = murmurHash(x)
+  elif A is char:
+    when not sHash2:
+      result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high)))
+    else:
+      #when nimvm:
+      #  result = hashVmImplChar(x, 0, x.high)
+      when true:
+        result = murmurHash(toOpenArrayByte(x, 0, x.high))
   else:
     result = 0
     for a in x:
@@ -574,17 +747,24 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash =
   runnableExamples:
     let a = [1, 2, 5, 1, 2, 6]
     doAssert hash(a, 0, 1) == hash(a, 3, 4)
-
   when A is byte:
-    when nimvm:
-      result = hashVmImplByte(aBuf, sPos, ePos)
+    maybeFailJS_Number()
+    when not sHash2:
+      result = cast[Hash](hashFarm(toOpenArray(aBuf, sPos, ePos)))
     else:
-      result = murmurHash(toOpenArray(aBuf, sPos, ePos))
+      #when nimvm:
+      #  result = hashVmImplByte(aBuf, sPos, ePos)
+      when true:
+        result = murmurHash(toOpenArray(aBuf, sPos, ePos))
   elif A is char:
-    when nimvm:
-      result = hashVmImplChar(aBuf, sPos, ePos)
+    maybeFailJS_Number()
+    when not sHash2:
+      result = cast[Hash](hashFarm(toOpenArrayByte(aBuf, sPos, ePos)))
     else:
-      result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos))
+      #when nimvm:
+      #  result = hashVmImplChar(aBuf, sPos, ePos)
+      when true:
+        result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos))
   else:
     for i in sPos .. ePos:
       result = result !& hash(aBuf[i])

lib/pure/lexbase.nim

@@ -104,9 +104,9 @@ proc fillBaseLexer(L: var BaseLexer, pos: int): int =
     result = 0
 
 proc handleCR*(L: var BaseLexer, pos: int): int =
-  ## Call this if you scanned over '\c' in the buffer; it returns the
+  ## Call this if you scanned over `'\c'` in the buffer; it returns the
   ## position to continue the scanning from. `pos` must be the position
-  ## of the '\c'.
+  ## of the `'\c'`.
   assert(L.buf[pos] == '\c')
   inc(L.lineNumber)
   result = fillBaseLexer(L, pos)
@@ -115,9 +115,9 @@ proc handleCR*(L: var BaseLexer, pos: int): int =
   L.lineStart = result
 
 proc handleLF*(L: var BaseLexer, pos: int): int =
-  ## Call this if you scanned over '\L' in the buffer; it returns the
+  ## Call this if you scanned over `'\L'` in the buffer; it returns the
   ## position to continue the scanning from. `pos` must be the position
-  ## of the '\L'.
+  ## of the `'\L'`.
   assert(L.buf[pos] == '\L')
   inc(L.lineNumber)
   result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result;

lib/pure/math.nim

@@ -58,6 +58,7 @@ import std/private/since
                        # of the standard library!
 
 import std/[bitops, fenv]
+import system/countbits_impl
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -155,7 +156,7 @@ func fac*(n: int): int =
 
 {.push checks: off, line_dir: off, stack_trace: off.}
 
-when defined(posix) and not defined(genode):
+when defined(posix) and not defined(genode) and not defined(macosx):
   {.passl: "-lm".}
 
 const
@@ -1229,40 +1230,42 @@ func gcd*[T](x, y: T): T =
     swap x, y
   abs x
 
-func gcd*(x, y: SomeInteger): SomeInteger =
-  ## Computes the greatest common (positive) divisor of `x` and `y`,
-  ## using the binary GCD (aka Stein's) algorithm.
-  ##
-  ## **See also:**
-  ## * `gcd func <#gcd,T,T>`_ for a float version
-  ## * `lcm func <#lcm,T,T>`_
-  runnableExamples:
-    doAssert gcd(12, 8) == 4
-    doAssert gcd(17, 63) == 1
-
-  when x is SomeSignedInt:
-    var x = abs(x)
-  else:
-    var x = x
-  when y is SomeSignedInt:
-    var y = abs(y)
-  else:
-    var y = y
-
-  if x == 0:
-    return y
-  if y == 0:
-    return x
-
-  let shift = countTrailingZeroBits(x or y)
-  y = y shr countTrailingZeroBits(y)
-  while x != 0:
-    x = x shr countTrailingZeroBits(x)
-    if y > x:
-      swap y, x
-    x -= y
-  y shl shift
-
+when useBuiltins:
+  ## this func uses bitwise comparisons from C compilers, which are not always available.
+  func gcd*(x, y: SomeInteger): SomeInteger =
+    ## Computes the greatest common (positive) divisor of `x` and `y`,
+    ## using the binary GCD (aka Stein's) algorithm.
+    ##
+    ## **See also:**
+    ## * `gcd func <#gcd,T,T>`_ for a float version
+    ## * `lcm func <#lcm,T,T>`_
+    runnableExamples:
+      doAssert gcd(12, 8) == 4
+      doAssert gcd(17, 63) == 1
+  
+    when x is SomeSignedInt:
+      var x = abs(x)
+    else:
+      var x = x
+    when y is SomeSignedInt:
+      var y = abs(y)
+    else:
+      var y = y
+  
+    if x == 0:
+      return y
+    if y == 0:
+      return x
+  
+    let shift = countTrailingZeroBits(x or y)
+    y = y shr countTrailingZeroBits(y)
+    while x != 0:
+      x = x shr countTrailingZeroBits(x)
+      if y > x:
+        swap y, x
+      x -= y
+    y shl shift
+  
 func gcd*[T](x: openArray[T]): T {.since: (1, 1).} =
   ## Computes the greatest common (positive) divisor of the elements of `x`.
   ##

lib/pure/memfiles.nim

@@ -35,12 +35,12 @@ proc newEIO(msg: string): ref IOError =
   new(result)
   result.msg = msg
 
-proc setFileSize(fh: FileHandle, newFileSize = -1): OSErrorCode =
-  ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1.
-  ## Space is only allocated if that is cheaper than writing to the file.  This
-  ## routine returns the last OSErrorCode found rather than raising to support
-  ## old rollback/clean-up code style. [ Should maybe move to std/osfiles. ]
-  if newFileSize == -1:
+proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode =
+  ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1,
+  ## allocating | freeing space from the file system.  This routine returns the
+  ## last OSErrorCode found rather than raising to support old rollback/clean-up
+  ## code style. [ Should maybe move to std/osfiles. ]
+  if newFileSize < 0 or newFileSize == oldSize:
     return
   when defined(windows):
     var sizeHigh = int32(newFileSize shr 32)
@@ -51,14 +51,18 @@ proc setFileSize(fh: FileHandle, newFileSize = -1): OSErrorCode =
         setEndOfFile(fh) == 0:
       result = lastErr
   else:
-    var e: cint # posix_fallocate truncates up when needed.
-    when declared(posix_fallocate):
-      while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR):
-        discard
-    if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1:
-      result = osLastError() # fallback arguable; Most portable, but allows SEGV
-    elif e != 0:
-      result = osLastError()
+    if newFileSize > oldSize: # grow the file
+      var e: cint # posix_fallocate truncates up when needed.
+      when declared(posix_fallocate):
+        while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR):
+          discard
+      if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1:
+        result = osLastError() # fallback arguable; Most portable BUT allows SEGV
+      elif e != 0:
+        result = osLastError()
+    else: # shrink the file
+      if ftruncate(fh.cint, newFileSize) == -1:
+        result = osLastError()
 
 type
   MemFile* = object      ## represents a memory mapped file
@@ -255,41 +259,31 @@ proc open*(filename: string, mode: FileMode = fmRead,
       flags = flags or O_CREAT or O_TRUNC
       var permissionsMode = S_IRUSR or S_IWUSR
       result.handle = open(filename, flags, permissionsMode)
+      if result.handle != -1:
+        if (let e = setFileSize(result.handle.FileHandle, newFileSize);
+            e != 0.OSErrorCode): fail(e, "error setting file size")
     else:
       result.handle = open(filename, flags)
 
     if result.handle == -1:
-      # XXX: errno is supposed to be set here
-      # Is there an exception that wraps it?
       fail(osLastError(), "error opening file")
 
-    if (let e = setFileSize(result.handle.FileHandle, newFileSize);
-        e != 0.OSErrorCode): fail(e, "error setting file size")
-
-    if mappedSize != -1:
-      result.size = mappedSize
-    else:
-      var stat: Stat
+    if mappedSize != -1: #XXX Logic here differs from `when windows` branch ..
+      result.size = mappedSize #.. which always fstats&Uses min(mappedSize, st).
+    else: # if newFileSize!=-1: result.size=newFileSize # if trust setFileSize
+      var stat: Stat  #^^.. BUT some FSes (eg. Linux HugeTLBfs) round to 2MiB.
       if fstat(result.handle, stat) != -1:
-        # XXX: Hmm, this could be unsafe
-        # Why is mmap taking int anyway?
-        result.size = int(stat.st_size)
+        result.size = stat.st_size.int # int may be 32-bit-unsafe for 2..<4 GiB
       else:
         fail(osLastError(), "error getting file size")
 
     result.flags = if mapFlags == cint(-1): MAP_SHARED else: mapFlags
-    #Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set
+    # Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set
     if int(result.flags and MAP_PRIVATE) == 0:
       result.flags = result.flags or MAP_SHARED
 
-    result.mem = mmap(
-      nil,
-      result.size,
-      if readonly: PROT_READ else: PROT_READ or PROT_WRITE,
-      result.flags,
-      result.handle,
-      offset)
-
+    let pr = if readonly: PROT_READ else: PROT_READ or PROT_WRITE
+    result.mem = mmap(nil, result.size, pr, result.flags, result.handle, offset)
     if result.mem == cast[pointer](MAP_FAILED):
       fail(osLastError(), "file mapping failed")
 
@@ -353,7 +347,7 @@ proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} =
       raise newException(IOError,
                          "Cannot resize MemFile opened with allowRemap=false")
     if newFileSize != f.size:
-      if (let e = setFileSize(f.handle.FileHandle, newFileSize);
+      if (let e = setFileSize(f.handle.FileHandle, newFileSize, f.size);
           e != 0.OSErrorCode): raiseOSError(e)
     when defined(linux): #Maybe NetBSD, too?
       # On Linux this can be over 100 times faster than a munmap,mmap cycle.

lib/pure/options.nim

@@ -117,9 +117,10 @@ proc option*[T](val: sink T): Option[T] {.inline.} =
     assert option[Foo](nil).isNone
     assert option(42).isSome
 
-  result.val = val
-  when T isnot SomePointer:
-    result.has = true
+  when T is SomePointer:
+    result = Option[T](val: val)
+  else:
+    result = Option[T](has: true, val: val)
 
 proc some*[T](val: sink T): Option[T] {.inline.} =
   ## Returns an `Option` that has the value `val`.
@@ -136,10 +137,9 @@ proc some*[T](val: sink T): Option[T] {.inline.} =
 
   when T is SomePointer:
     assert not val.isNil
-    result.val = val
+    result = Option[T](val: val)
   else:
-    result.has = true
-    result.val = val
+    result = Option[T](has: true, val: val)
 
 proc none*(T: typedesc): Option[T] {.inline.} =
   ## Returns an `Option` for this type that has no value.

lib/pure/os.nim

@@ -692,7 +692,10 @@ proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdT
 
 proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} =
   ## Sleeps `milsecs` milliseconds.
+  ## A negative `milsecs` causes sleep to return immediately.
   when defined(windows):
+    if milsecs < 0:
+      return  # fixes #23732
     winlean.sleep(int32(milsecs))
   else:
     var a, b: Timespec

lib/pure/osproc.nim

@@ -1416,6 +1416,8 @@ elif not defined(useNimRtl):
           tmspec.tv_nsec = (timeout * 1_000_000)
 
         try:
+          if not running(p):
+            return exitStatusLikeShell(p.exitStatus)
           if clock_gettime(CLOCK_REALTIME, stspec) == -1:
             raiseOSError(osLastError())
           while true:

lib/pure/parseopt.nim

@@ -176,6 +176,7 @@
 
 include "system/inclrtl"
 
+import std/strutils
 import std/os
 
 type
@@ -249,9 +250,20 @@ proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {},
       result.cmds[i] = cmdline[i]
   else:
     when declared(paramCount):
-      result.cmds = newSeq[string](paramCount())
-      for i in countup(1, paramCount()):
-        result.cmds[i-1] = paramStr(i)
+      when defined(nimscript):
+        var ctr = 0
+        var firstNimsFound = false
+        for i in countup(0, paramCount()):
+          if firstNimsFound: 
+            result.cmds[ctr] = paramStr(i)
+            inc ctr, 1
+          if paramStr(i).endsWith(".nims") and not firstNimsFound:
+            firstNimsFound = true 
+            result.cmds = newSeq[string](paramCount()-i)
+      else:
+        result.cmds = newSeq[string](paramCount())
+        for i in countup(1, paramCount()):
+          result.cmds[i-1] = paramStr(i)
     else:
       # we cannot provide this for NimRtl creation on Posix, because we can't
       # access the command line arguments then!

lib/pure/parseutils.nim

@@ -460,6 +460,8 @@ proc parseBiggestInt*(s: openArray[char], number: var BiggestInt): int {.
     var res: BiggestInt
     doAssert parseBiggestInt("9223372036854775807", res) == 19
     doAssert res == 9223372036854775807
+    doAssert parseBiggestInt("-2024_05_09", res) == 11
+    doAssert res == -20240509
   var res = BiggestInt(0)
   # use 'res' for exception safety (don't write to 'number' in case of an
   # overflow exception):
@@ -474,10 +476,8 @@ proc parseInt*(s: openArray[char], number: var int): int {.
   ## `ValueError` is raised if the parsed integer is out of the valid range.
   runnableExamples:
     var res: int
-    doAssert parseInt("2019", res, 0) == 4
-    doAssert res == 2019
-    doAssert parseInt("2019", res, 2) == 2
-    doAssert res == 19
+    doAssert parseInt("-2024_05_02", res) == 11
+    doAssert res == -20240502
   var res = BiggestInt(0)
   result = parseBiggestInt(s, res)
   when sizeof(int) <= 4:
@@ -992,6 +992,10 @@ proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {.noSid
     var res: BiggestInt
     doAssert parseBiggestInt("9223372036854775807", res, 0) == 19
     doAssert res == 9223372036854775807
+    doAssert parseBiggestInt("-2024_05_09", res) == 11
+    doAssert res == -20240509
+    doAssert parseBiggestInt("-2024_05_02", res, 7) == 4
+    doAssert res == 502
   parseBiggestInt(s.toOpenArray(start, s.high), number)
 
 proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raises: [ValueError].} =
@@ -1000,10 +1004,10 @@ proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raise
   ## `ValueError` is raised if the parsed integer is out of the valid range.
   runnableExamples:
     var res: int
-    doAssert parseInt("2019", res, 0) == 4
-    doAssert res == 2019
-    doAssert parseInt("2019", res, 2) == 2
-    doAssert res == 19
+    doAssert parseInt("-2024_05_02", res) == 11
+    doAssert res == -20240502
+    doAssert parseInt("-2024_05_02", res, 7) == 4
+    doAssert res == 502
   parseInt(s.toOpenArray(start, s.high), number)
 
 

lib/pure/strmisc.nim

@@ -27,20 +27,17 @@ func expandTabs*(s: string, tabSize: int = 8): string =
     doAssert expandTabs("a\tb\n\txy\t", 3) == "a  b\n   xy "
 
   result = newStringOfCap(s.len + s.len shr 2)
-  var pos = 0
 
   template addSpaces(n) =
-    for j in 0 ..< n:
+    for _ in 1..n:
       result.add(' ')
-      pos += 1
+    pos += n
 
-  for i in 0 ..< len(s):
-    let c = s[i]
+  var pos = 0
+  let denominator = if tabSize > 0: tabSize else: 1
+  for c in s:
     if c == '\t':
-      let
-        denominator = if tabSize > 0: tabSize else: 1
-        numSpaces = tabSize - pos mod denominator
-
+      let numSpaces = tabSize - pos mod denominator
       addSpaces(numSpaces)
     else:
       result.add(c)

lib/pure/strutils.nim

@@ -1307,7 +1307,7 @@ func parseEnum*[T: enum](s: string): T =
   ## type contains multiple fields with the same string value.
   ##
   ## Raises `ValueError` for an invalid value in `s`. The comparison is
-  ## done in a style insensitive way.
+  ## done in a style insensitive way (first letter is still case-sensitive).
   runnableExamples:
     type
       MyEnum = enum
@@ -1327,7 +1327,7 @@ func parseEnum*[T: enum](s: string, default: T): T =
   ## type contains multiple fields with the same string value.
   ##
   ## Uses `default` for an invalid value in `s`. The comparison is done in a
-  ## style insensitive way.
+  ## style insensitive way (first letter is still case-sensitive).
   runnableExamples:
     type
       MyEnum = enum

lib/std/paths.nim

@@ -9,7 +9,7 @@ export osseps
 import std/envvars
 import std/private/osappdirs
 
-import std/pathnorm
+import std/[pathnorm, hashes, sugar, strutils]
 
 from std/private/ospaths2 import  joinPath, splitPath,
                                   ReadDirEffect, WriteDirEffect,
@@ -25,6 +25,13 @@ export ReadDirEffect, WriteDirEffect
 type
   Path* = distinct string
 
+func hash*(x: Path): Hash =
+  let x = x.string.dup(normalizePath)
+  if FileSystemCaseSensitive:
+    result = x.hash
+  else:
+    result = x.toLowerAscii.hash
+
 template `$`*(x: Path): string =
   string(x)
 

lib/std/private/ossymlinks.nim

@@ -66,11 +66,13 @@ proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} =
   when defined(windows) or defined(nintendoswitch):
     result = symlinkPath
   else:
-    result = newString(maxSymlinkLen)
-    var len = readlink(symlinkPath, result.cstring, maxSymlinkLen)
-    if len < 0:
-      raiseOSError(osLastError(), symlinkPath)
-    if len > maxSymlinkLen:
-      result = newString(len+1)
-      len = readlink(symlinkPath, result.cstring, len)
-    setLen(result, len)
+    var bufLen = 1024
+    while true:
+      result = newString(bufLen)
+      let len = readlink(symlinkPath.cstring, result.cstring, bufLen)
+      if len < 0:
+        raiseOSError(osLastError(), symlinkPath)
+      if len < bufLen:
+        result.setLen(len)
+        break
+      bufLen = bufLen shl 1

lib/std/syncio.nim

@@ -874,7 +874,7 @@ proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} =
   var f: File = nil
   if open(f, filename, fmWrite):
     try:
-      f.writeBuffer(unsafeAddr content[0], content.len)
+      discard f.writeBuffer(unsafeAddr content[0], content.len)
     finally:
       close(f)
   else:

lib/std/tasks.nim

@@ -110,6 +110,19 @@ template addAllNode(assignParam: NimNode, procParam: NimNode) =
   tempAssignList.add newLetStmt(tempNode, newDotExpr(objTemp, formalParams[i][0]))
   scratchRecList.add newIdentDefs(newIdentNode(formalParams[i][0].strVal), assignParam)
 
+proc analyseRootSym(s: NimNode): NimNode =
+  result = s
+  while true:
+    case result.kind
+    of nnkBracketExpr, nnkDerefExpr, nnkHiddenDeref,
+        nnkAddr, nnkHiddenAddr,
+        nnkObjDownConv, nnkObjUpConv:
+      result = result[0]
+    of nnkDotExpr, nnkCheckedFieldExpr, nnkHiddenStdConv, nnkHiddenSubConv:
+      result = result[1]
+    else:
+      break
+
 macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkCallStrLit}): Task =
   ## Converts the call and its arguments to `Task`.
   runnableExamples:
@@ -121,11 +134,14 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
   let retType = getTypeInst(e)
   let returnsVoid = retType.typeKind == ntyVoid
 
+  let rootSym = analyseRootSym(e[0])
+  expectKind rootSym, nnkSym
+
   when compileOption("threads"):
-    if not isGcSafe(e[0]):
+    if not isGcSafe(rootSym):
       error("'toTask' takes a GC safe call expression", e)
 
-  if hasClosure(e[0]):
+  if hasClosure(rootSym):
     error("closure call is not allowed", e)
 
   if e.len > 1:
@@ -209,7 +225,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
     let funcCall = newCall(e[0], callNode)
     functionStmtList.add tempAssignList
 
-    let funcName = genSym(nskProc, e[0].strVal)
+    let funcName = genSym(nskProc, rootSym.strVal)
     let destroyName = genSym(nskProc, "destroyScratch")
     let objTemp2 = genSym(ident = "obj")
     let tempNode = quote("@") do:
@@ -241,7 +257,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
       Task(callback: `funcName`, args: `scratchIdent`, destroy: `destroyName`)
   else:
     let funcCall = newCall(e[0])
-    let funcName = genSym(nskProc, e[0].strVal)
+    let funcName = genSym(nskProc, rootSym.strVal)
 
     if returnsVoid:
       result = quote do:

lib/system.nim

@@ -2102,12 +2102,14 @@ when not defined(js):
     proc cstringArrayToSeq*(a: cstringArray, len: Natural): seq[string] =
       ## Converts a `cstringArray` to a `seq[string]`. `a` is supposed to be
       ## of length `len`.
+      if a == nil: return @[]
       newSeq(result, len)
       for i in 0..len-1: result[i] = $a[i]
 
     proc cstringArrayToSeq*(a: cstringArray): seq[string] =
       ## Converts a `cstringArray` to a `seq[string]`. `a` is supposed to be
       ## terminated by `nil`.
+      if a == nil: return @[]
       var L = 0
       while a[L] != nil: inc(L)
       result = cstringArrayToSeq(a, L)

lib/system/alloc.nim

@@ -93,8 +93,6 @@ type
     freeList: ptr FreeCell
     free: int            # how many bytes remain
     acc: int             # accumulator for small object allocation
-    when defined(gcDestructors):
-      sharedFreeList: ptr FreeCell # make no attempt at avoiding false sharing for now for this object field
     data {.align: MemAlign.}: UncheckedArray[byte]      # start of usable memory
 
   BigChunk = object of BaseChunk # not necessarily > PageSize!
@@ -109,7 +107,9 @@ type
   MemRegion = object
     when not defined(gcDestructors):
       minLargeObj, maxLargeObj: int
-    freeSmallChunks: array[0..max(1,SmallChunkSize div MemAlign-1), PSmallChunk]
+    freeSmallChunks: array[0..max(1, SmallChunkSize div MemAlign-1), PSmallChunk]
+    when defined(gcDestructors):
+      sharedFreeLists: array[0..max(1, SmallChunkSize div MemAlign-1), ptr FreeCell]
     flBitmap: uint32
     slBitmap: array[RealFli, uint32]
     matrix: array[RealFli, array[MaxSli, PBigChunk]]
@@ -777,8 +777,10 @@ when defined(gcDestructors):
     sysAssert c.next == nil, "c.next pointer must be nil"
     atomicPrepend a.sharedFreeListBigChunks, c
 
-  proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell) {.inline.} =
-    atomicPrepend c.sharedFreeList, f
+  proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell; size: int) {.inline.} =
+    atomicPrepend c.owner.sharedFreeLists[size], f
+
+  const MaxSteps = 20
 
   proc compensateCounters(a: var MemRegion; c: PSmallChunk; size: int) =
     # rawDealloc did NOT do the usual:
@@ -787,31 +789,27 @@ when defined(gcDestructors):
     # Well, not for the entire list, but for `max` elements of the list because
     # we split the list in order to achieve bounded response times.
     var it = c.freeList
-    var x = 0
-    var maxIters = 20 # make it time-bounded
+    var total = 0
     while it != nil:
-      if maxIters == 0:
-        let rest = it.next.loada
-        if rest != nil:
-          it.next.storea nil
-          addToSharedFreeList(c, rest)
-        break
-      inc x, size
-      it = it.next.loada
-      dec maxIters
-    inc(c.free, x)
-    dec(a.occ, x)
+      inc total, size
+      let chunk = cast[PSmallChunk](pageAddr(it))
+      inc(chunk.free, size)
+      it = it.next
+    dec(a.occ, total)
 
   proc freeDeferredObjects(a: var MemRegion; root: PBigChunk) =
     var it = root
-    var maxIters = 20 # make it time-bounded
+    var maxIters = MaxSteps # make it time-bounded
     while true:
+      let rest = it.next.loada
+      it.next.storea nil
+      deallocBigChunk(a, cast[PBigChunk](it))
       if maxIters == 0:
-        let rest = it.next.loada
-        it.next.storea nil
-        addToSharedFreeListBigChunks(a, rest)
+        if rest != nil:
+          addToSharedFreeListBigChunks(a, rest)
+          sysAssert a.sharedFreeListBigChunks != nil, "re-enqueing failed"
         break
-      it = it.next.loada
+      it = rest
       dec maxIters
       if it == nil: break
 
@@ -835,8 +833,6 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
       sysAssert c.size == PageSize, "rawAlloc 3"
       c.size = size
       c.acc = size
-      when defined(gcDestructors):
-        c.sharedFreeList = nil
       c.free = SmallChunkSize - smallChunkOverhead() - size
       sysAssert c.owner == addr(a), "rawAlloc: No owner set!"
       c.next = nil
@@ -853,10 +849,11 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
       when defined(gcDestructors):
         if c.freeList == nil:
           when hasThreadSupport:
-            c.freeList = atomicExchangeN(addr c.sharedFreeList, nil, ATOMIC_RELAXED)
+            # Steal the entire list from `sharedFreeList`:
+            c.freeList = atomicExchangeN(addr a.sharedFreeLists[s], nil, ATOMIC_RELAXED)
           else:
-            c.freeList = c.sharedFreeList
-            c.sharedFreeList = nil
+            c.freeList = a.sharedFreeLists[s]
+            a.sharedFreeLists[s] = nil
           compensateCounters(a, c, size)
       if c.freeList == nil:
         sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize,
@@ -868,7 +865,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
         when not defined(gcDestructors):
           sysAssert(c.freeList.zeroField == 0, "rawAlloc 8")
         c.freeList = c.freeList.next
-      dec(c.free, size)
+      dec(cast[PSmallChunk](pageAddr(result)).free, size)
       sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 9")
       sysAssert(allocInv(a), "rawAlloc: end c != nil")
     sysAssert(allocInv(a), "rawAlloc: before c.free < size")
@@ -907,7 +904,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
     trackSize(c.size)
   sysAssert(isAccessible(a, result), "rawAlloc 14")
   sysAssert(allocInv(a), "rawAlloc: end")
-  when logAlloc: cprintf("var pointer_%p = alloc(%ld)\n", result, requestedSize)
+  when logAlloc: cprintf("var pointer_%p = alloc(%ld) # %p\n", result, requestedSize, addr a)
 
 proc rawAlloc0(a: var MemRegion, requestedSize: int): pointer =
   result = rawAlloc(a, requestedSize)
@@ -923,7 +920,7 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
   if isSmallChunk(c):
     # `p` is within a small chunk:
     var c = cast[PSmallChunk](c)
-    var s = c.size
+    let s = c.size
     #       ^ We might access thread foreign storage here.
     # The other thread cannot possibly free this block as it's still alive.
     var f = cast[ptr FreeCell](p)
@@ -951,18 +948,21 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
         inc(c.free, s)
       else:
         inc(c.free, s)
-        if c.free == SmallChunkSize-smallChunkOverhead():
+        if c.free == SmallChunkSize-smallChunkOverhead() and a.freeSmallChunks[s div MemAlign] == c:
           listRemove(a.freeSmallChunks[s div MemAlign], c)
           c.size = SmallChunkSize
           freeBigChunk(a, cast[PBigChunk](c))
     else:
+      when logAlloc: cprintf("dealloc(pointer_%p) # SMALL FROM %p CALLER %p\n", p, c.owner, addr(a))
+
       when defined(gcDestructors):
-        addToSharedFreeList(c, f)
+        addToSharedFreeList(c, f, s div MemAlign)
     sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %%
                s == 0, "rawDealloc 2")
   else:
     # set to 0xff to check for usage after free bugs:
     when overwriteFree: nimSetMem(p, -1'i32, c.size -% bigChunkOverhead())
+    when logAlloc: cprintf("dealloc(pointer_%p) # BIG %p\n", p, c.owner)
     when defined(gcDestructors):
       if c.owner == addr(a):
         deallocBigChunk(a, cast[PBigChunk](c))
@@ -970,8 +970,9 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
         addToSharedFreeListBigChunks(c.owner[], cast[PBigChunk](c))
     else:
       deallocBigChunk(a, cast[PBigChunk](c))
+
   sysAssert(allocInv(a), "rawDealloc: end")
-  when logAlloc: cprintf("dealloc(pointer_%p)\n", p)
+  #when logAlloc: cprintf("dealloc(pointer_%p)\n", p)
 
 when not defined(gcDestructors):
   proc isAllocatedPtr(a: MemRegion, p: pointer): bool =

lib/system/indices.nim

@@ -80,8 +80,6 @@ proc `[]`*[T, U: Ordinal](s: string, x: HSlice[T, U]): string {.inline, systemRa
   ##   var s = "abcdef"
   ##   assert s[1..3] == "bcd"
   ##   ```
-  # Workaround bug #22852
-  result = ""
   let a = s ^^ x.a
   let L = (s ^^ x.b) - a + 1
   result = newString(L)
@@ -114,11 +112,8 @@ proc `[]`*[Idx, T; U, V: Ordinal](a: array[Idx, T], x: HSlice[U, V]): seq[T] {.s
   ##   ```
   let xa = a ^^ x.a
   let L = (a ^^ x.b) - xa + 1
-  # Workaround bug #22852:
-  result = newSeq[T](if L < 0: 0 else: L)
+  result = newSeq[T](L)
   for i in 0..<L: result[i] = a[Idx(i + xa)]
-  # Workaround bug #22852
-  discard Natural(L)
 
 proc `[]=`*[Idx, T; U, V: Ordinal](a: var array[Idx, T], x: HSlice[U, V], b: openArray[T]) {.systemRaisesDefect.} =
   ## Slice assignment for arrays.

lib/system/orc.nim

@@ -146,7 +146,7 @@ proc unregisterCycle(s: Cell) =
   let idx = s.rootIdx-1
   when false:
     if idx >= roots.len or idx < 0:
-      cprintf("[Bug!] %ld\n", idx)
+      cprintf("[Bug!] %ld %ld\n", idx, roots.len)
       rawQuit 1
   roots.d[idx] = roots.d[roots.len-1]
   roots.d[idx][0].rootIdx = idx+1
@@ -303,6 +303,14 @@ proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) =
         t.setColor(colBlack)
         trace(t, desc, j)
 
+const
+  defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128
+
+when defined(nimStressOrc):
+  const rootsThreshold = 10 # broken with -d:nimStressOrc: 10 and for havlak iterations 1..8
+else:
+  var rootsThreshold {.threadvar.}: int
+
 proc collectCyclesBacon(j: var GcEnv; lowMark: int) =
   # pretty direct translation from
   # https://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon01Concurrent.pdf
@@ -341,22 +349,25 @@ proc collectCyclesBacon(j: var GcEnv; lowMark: int) =
     s.rootIdx = 0
     collectColor(s, roots.d[i][1], colToCollect, j)
 
+  # Bug #22927: `free` calls destructors which can append to `roots`.
+  # We protect against this here by setting `roots.len` to 0 and also
+  # setting the threshold so high that no cycle collection can be triggered
+  # until we are out of this critical section:
+  when not defined(nimStressOrc):
+    let oldThreshold = rootsThreshold
+    rootsThreshold = high(int)
+  roots.len = 0
+
   for i in 0 ..< j.toFree.len:
     when orcLeakDetector:
       writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1])
     free(j.toFree.d[i][0], j.toFree.d[i][1])
 
+  when not defined(nimStressOrc):
+    rootsThreshold = oldThreshold
+
   inc j.freed, j.toFree.len
   deinit j.toFree
-  #roots.len = 0
-
-const
-  defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128
-
-when defined(nimStressOrc):
-  const rootsThreshold = 10 # broken with -d:nimStressOrc: 10 and for havlak iterations 1..8
-else:
-  var rootsThreshold {.threadvar.}: int
 
 when defined(nimOrcStats):
   var freedCyclicObjects {.threadvar.}: int
@@ -396,7 +407,8 @@ proc collectCycles() =
     collectCyclesBacon(j, 0)
 
   deinit j.traceStack
-  deinit roots
+  if roots.len == 0:
+    deinit roots
 
   when not defined(nimStressOrc):
     # compute the threshold based on the previous history

lib/system/sysstr.nim

@@ -220,7 +220,10 @@ proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} =
 proc setLengthStr(s: NimString, newLen: int): NimString {.compilerRtl.} =
   let n = max(newLen, 0)
   if s == nil:
-    result = mnewString(n)
+    if n == 0:
+      return s
+    else:
+      result = mnewString(n)
   elif n <= s.space:
     result = s
   else:
@@ -301,7 +304,10 @@ proc setLengthSeqV2(s: PGenericSeq, typ: PNimType, newLen: int): PGenericSeq {.
     compilerRtl.} =
   sysAssert typ.kind == tySequence, "setLengthSeqV2: type is not a seq"
   if s == nil:
-    result = cast[PGenericSeq](newSeq(typ, newLen))
+    if newLen == 0:
+      result = s
+    else:
+      result = cast[PGenericSeq](newSeq(typ, newLen))
   else:
     let elemSize = typ.base.size
     let elemAlign = typ.base.align

nimdoc/extlinks/project/expected/_._/util.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>nimdoc/extlinks/util</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -97,8 +101,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/extlinks/project/expected/doc/manual.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>Nothing User Manual</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -37,8 +41,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/extlinks/project/expected/main.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>nimdoc/extlinks/project/main</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -134,8 +138,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/extlinks/project/expected/sub/submodule.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>nimdoc/extlinks/project/sub/submodule</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -123,8 +127,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/extlinks/project/expected/theindex.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>Index</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -51,8 +55,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/rst2html/expected/rst_examples.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>Not a Nim Manual</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -266,8 +270,5 @@ stmt = IND{&gt;} stmt ^+ IND{=} DED  # list of statements
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/test_doctype/expected/test_doctype.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>nimdoc/test_doctype/test_doctype</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -75,8 +79,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/test_out_index_dot_html/expected/index.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>nimdoc/test_out_index_dot_html/foo</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -97,8 +101,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/test_out_index_dot_html/expected/theindex.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>Index</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -35,8 +39,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/testproject/expected/subdir/subdir_b/utils.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>subdir/subdir_b/utils</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -606,8 +610,5 @@ Ref. <a class="reference internal nimdoc" title="proc `[]`[T](x: G[T]): T" href=
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/testproject/expected/testproject.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>testproject</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -1252,8 +1256,5 @@ bar
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimdoc/testproject/expected/theindex.html

@@ -7,6 +7,10 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>Index</title>
 
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
 <!-- Favicon -->
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 <link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
@@ -423,8 +427,5 @@
     </div>
   </div>
   
-  <!-- Google fonts -->
-  <link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
-  <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </body>
 </html>

nimsuggest/nimsuggest.nim

@@ -134,7 +134,8 @@ const
   #List of currently supported capabilities. So lang servers/ides can iterate over and check for what's enabled
   Capabilities = [
     "con", #current NimSuggest supports the `con` commmand
-    "exceptionInlayHints"
+    "exceptionInlayHints",
+    "unknownFile", #current NimSuggest can handle unknown files
   ]
 
 proc parseQuoted(cmd: string; outp: var string; start: int): int =
@@ -758,15 +759,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
 
   if gMode != mstdin:
     conf.writelnHook = proc (msg: string) = discard
-  # Find Nim's prefix dir.
-  let binaryPath = findExe("nim")
-  if binaryPath == "":
-    raise newException(IOError,
-        "Cannot find Nim standard library: Nim compiler not in PATH")
-  conf.prefixDir = AbsoluteDir binaryPath.splitPath().head.parentDir()
-  if not dirExists(conf.prefixDir / RelativeDir"lib"):
-    conf.prefixDir = AbsoluteDir""
-
+  conf.prefixDir = conf.getPrefixDir()
   #msgs.writelnHook = proc (line: string) = log(line)
   myLog("START " & conf.projectFull.string)
 
@@ -1065,10 +1058,6 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile,
   var fileIndex: FileIndex
 
   if not (cmd in {ideRecompile, ideGlobalSymbols}):
-    if not fileInfoKnown(conf, file):
-      myLog fmt "{file} is unknown, returning no results"
-      return
-
     fileIndex = fileInfoIdx(conf, file)
     msgs.setDirtyFile(
       conf,
@@ -1344,13 +1333,7 @@ else:
       conf.writelnHook = proc (msg: string) = discard
     # Find Nim's prefix dir.
     if nimPath == "":
-      let binaryPath = findExe("nim")
-      if binaryPath == "":
-        raise newException(IOError,
-            "Cannot find Nim standard library: Nim compiler not in PATH")
-      conf.prefixDir = AbsoluteDir binaryPath.splitPath().head.parentDir()
-      if not dirExists(conf.prefixDir / RelativeDir"lib"):
-        conf.prefixDir = AbsoluteDir""
+      conf.prefixDir = conf.getPrefixDir()
     else:
       conf.prefixDir = AbsoluteDir nimPath
 

testament/important_packages.nim

@@ -63,10 +63,13 @@ pkg "criterion", allowFailure = true # needs testing binary
 pkg "datamancer"
 pkg "dashing", "nim c tests/functional.nim"
 pkg "delaunay"
+pkg "dnsclient", allowFailure = true # super fragile
 pkg "docopt"
+pkg "dotenv"
 # when defined(linux): pkg "drchaos"
 pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl"
 pkg "elvis"
+pkg "faststreams"
 pkg "fidget"
 pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 
 pkg "fusion"
@@ -79,10 +82,13 @@ pkg "gnuplot", "nim c gnuplot.nim"
   # pending https://github.com/nim-lang/Nim/issues/16509
 pkg "hts", "nim c -o:htss src/hts.nim"
 pkg "httpauth"
+pkg "httputils"
 pkg "illwill", "nimble examples"
 pkg "inim"
 pkg "itertools", "nim doc src/itertools.nim"
 pkg "iterutils"
+pkg "json_rpc"
+pkg "json_serialization"
 pkg "jstin"
 pkg "karax", "nim c -r tests/tester.nim"
 pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree"
@@ -124,6 +130,7 @@ pkg "nimwc", "nim c nimwc.nim"
 pkg "nimx", "nim c test/main.nim", allowFailure = true
 pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter"
 pkg "norm", "testament r tests/common/tmodel.nim"
+pkg "normalize"
 pkg "npeg", "nimble testarc"
 pkg "numericalnim", "nimble nimCI"
 pkg "optionsutils"
@@ -134,9 +141,9 @@ pkg "pixie"
 pkg "plotly", "nim c examples/all.nim"
 pkg "pnm"
 pkg "polypbren"
+pkg "presto"
 pkg "prologue", "nimble tcompile"
 pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim"
-pkg "pylib"
 pkg "rbtree"
 pkg "react", "nimble example"
 pkg "regex", "nim c src/regex"
@@ -145,6 +152,7 @@ pkg "RollingHash", "nim c -r tests/test_cyclichash.nim"
 pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim"
 pkg "sdl1", "nim c -r src/sdl.nim"
 pkg "sdl2_nim", "nim c -r sdl2/sdl.nim"
+pkg "serialization"
 pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4"
 pkg "sim"
 pkg "smtp", "nimble compileExample"
@@ -163,18 +171,22 @@ pkg "templates"
 pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git"
 pkg "terminaltables", "nim c src/terminaltables.nim"
 pkg "termstyle", "nim c -r termstyle.nim"
+pkg "testutils"
 pkg "timeit"
 pkg "timezones"
 pkg "tiny_sqlite"
 pkg "unicodedb", "nim c -d:release -r tests/tests.nim"
 pkg "unicodeplus", "nim c -d:release -r tests/tests.nim"
 pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union"
+pkg "unittest2"
 pkg "unpack"
 pkg "weave", "nimble test_gc_arc", useHead = true
+pkg "websock"
 pkg "websocket", "nim c websocket.nim"
-pkg "winim", "nim c winim.nim"
+# pkg "winim", allowFailure = true
 pkg "with"
 pkg "ws", allowFailure = true
 pkg "yaml"
 pkg "zero_functional", "nim c -r test.nim"
 pkg "zippy"
+pkg "zxcvbn"

tests/alloc/tmembug.nim

@@ -0,0 +1,54 @@
+discard """
+  joinable: false
+"""
+
+import std / [atomics, strutils, sequtils]
+
+type
+  BackendMessage* = object
+    field*: seq[int]
+
+var
+  chan1: Channel[BackendMessage]
+  chan2: Channel[BackendMessage]
+
+chan1.open()
+chan2.open()
+
+proc routeMessage*(msg: BackendMessage) =
+  discard chan2.trySend(msg)
+
+var
+  recv: Thread[void]
+  stopToken: Atomic[bool]
+
+proc recvMsg() =
+  while not stopToken.load(moRelaxed):
+    let resp = chan1.tryRecv()
+    if resp.dataAvailable:
+      routeMessage(resp.msg)
+      echo "child consumes ", formatSize getOccupiedMem()
+
+createThread[void](recv, recvMsg)
+
+const MESSAGE_COUNT = 100
+
+proc main() =
+  let msg: BackendMessage = BackendMessage(field: (0..500).toSeq())
+  for j in 0..0: #100:
+    echo "New iteration"
+
+    for _ in 1..MESSAGE_COUNT:
+      chan1.send(msg)
+    echo "After sending"
+
+    var counter = 0
+    while counter < MESSAGE_COUNT:
+      let resp = recv(chan2)
+      counter.inc
+    echo "After receiving ", formatSize getOccupiedMem()
+
+  stopToken.store true, moRelaxed
+  joinThreads(recv)
+
+main()

tests/alloc/tmembug2.nim

@@ -0,0 +1,58 @@
+discard """
+  disabled: "true"
+"""
+
+import std / [atomics, strutils, sequtils, isolation]
+
+import threading / channels
+
+type
+  BackendMessage* = object
+    field*: seq[int]
+
+const MESSAGE_COUNT = 100
+
+var
+  chan1 = newChan[BackendMessage](MESSAGE_COUNT*2)
+  chan2 = newChan[BackendMessage](MESSAGE_COUNT*2)
+
+#chan1.open()
+#chan2.open()
+
+proc routeMessage*(msg: BackendMessage) =
+  var m = isolate(msg)
+  discard chan2.trySend(m)
+
+var
+  thr: Thread[void]
+  stopToken: Atomic[bool]
+
+proc recvMsg() =
+  while not stopToken.load(moRelaxed):
+    var resp: BackendMessage
+    if chan1.tryRecv(resp):
+      #if resp.dataAvailable:
+      routeMessage(resp)
+      echo "child consumes ", formatSize getOccupiedMem()
+
+createThread[void](thr, recvMsg)
+
+proc main() =
+  let msg: BackendMessage = BackendMessage(field: (0..5).toSeq())
+  for j in 0..100:
+    echo "New iteration"
+
+    for _ in 1..MESSAGE_COUNT:
+      chan1.send(msg)
+    echo "After sending"
+
+    var counter = 0
+    while counter < MESSAGE_COUNT:
+      let resp = recv(chan2)
+      counter.inc
+    echo "After receiving ", formatSize getOccupiedMem()
+
+  stopToken.store true, moRelaxed
+  joinThreads(thr)
+
+main()