split PackedModule into reader and writer

[backport] fixes #23711 ; C code contains backtick`gensym (#23716 )
fixes #23711
2026-09-01 11:23:40 +00:00 · 2024-06-19 22:10:00 +08:00 · 2024-06-19 08:33:38 +02:00 · 2024-06-19 06:49:57 +02:00 · 2024-06-19 08:26:15 +08:00 · 2024-06-18 22:41:22 +02:00
62 changed files with 1190 additions and 408 deletions
--- a/changelog.md
+++ b/changelog.md
@@ -17,6 +17,8 @@

 - `bindMethod` in `std/jsffi` is deprecated, don't use it with closures.

+- JS backend now supports lambda lifting for closures. Use `--legacy:jsNoLambdaLifting` to emulate old behavior.
+
 ## Standard library additions and changes

 [//]: # "Changes:"
@@ -37,6 +39,9 @@ slots when enlarging a sequence.
  objects the cyclic collector did free. If the number is zero that is a strong indicator that you can use `--mm:arc`
  instead of `--mm:orc`.
 - A `$` template is provided for `Path` in `std/paths`.
+- `nimPreviewHashFarm` has been added to `lib/pure/hashes.nim` to default to a
+64-bit string `Hash` (based upon Google's Farm Hash) which is also faster than
+the present one.  At present, this is incompatible with `--jsbigint=off` mode.

 [//]: # "Deprecations:"

--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -41,7 +41,7 @@ type
  TNodeKinds* = set[TNodeKind]

 type
-  TSymFlag* = enum    # 51 flags!
+  TSymFlag* = enum    # 52 flags!
    sfUsed,           # read access of sym (for warnings) or simply used
    sfExported,       # symbol is exported from module
    sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -126,6 +126,7 @@ type
    sfByCopy          # param is marked as pass bycopy
    sfMember          # proc is a C++ member of a type
    sfCodegenDecl     # type, proc, global or proc param is marked as codegenDecl
+    sfWasGenSym       # symbol was 'gensym'ed

  TSymFlags* = set[TSymFlag]

@@ -331,7 +332,7 @@ type
    nfOpenSym # node is a captured sym but can be overriden by local symbols

  TNodeFlags* = set[TNodeFlag]
-  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 48)
+  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 47)
    tfVarargs,        # procedure has C styled varargs
                      # tyArray type represeting a varargs list
    tfNoSideEffect,   # procedure type does not allow side effects
@@ -403,7 +404,6 @@ type
    tfIsOutParam
    tfSendable
    tfImplicitStatic
-    tfTrackedProc     # used for delayedEffects

  TTypeFlags* = set[TTypeFlag]

--- a/compiler/ccgcalls.nim
+++ b/compiler/ccgcalls.nim
@@ -76,6 +76,23 @@ proc isHarmlessStore(p: BProc; canRaise: bool; d: TLoc): bool =
  else:
    result = false

+proc cleanupTemp(p: BProc; returnType: PType, tmp: TLoc): bool =
+  if returnType.kind in {tyVar, tyLent}:
+    # we don't need to worry about var/lent return types
+    result = false
+  elif hasDestructor(returnType) and getAttachedOp(p.module.g.graph, returnType, attachedDestructor) != nil:
+    let dtor = getAttachedOp(p.module.g.graph, returnType, attachedDestructor)
+    var op = initLocExpr(p, newSymNode(dtor))
+    var callee = rdLoc(op)
+    let destroy = if dtor.typ.firstParamType.kind == tyVar:
+        callee & "(&" & rdLoc(tmp) & ")"
+      else:
+        callee & "(" & rdLoc(tmp) & ")"
+    raiseExitCleanup(p, destroy)
+    result = true
+  else:
+    result = false
+
 proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
               callee, params: Rope) =
  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
@@ -128,18 +145,25 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
            if canRaise: raiseExit(p)

      elif isHarmlessStore(p, canRaise, d):
-        if d.k == locNone: d = getTemp(p, typ.returnType)
+        var useTemp = false
+        if d.k == locNone:
+          useTemp = true
+          d = getTemp(p, typ.returnType)
        assert(d.t != nil)        # generate an assignment to d:
        var list = initLoc(locCall, d.lode, OnUnknown)
        list.r = pl
        genAssignment(p, d, list, flags) # no need for deep copying
-        if canRaise: raiseExit(p)
+        if canRaise:
+          if not (useTemp and cleanupTemp(p, typ.returnType, d)):
+            raiseExit(p)
      else:
        var tmp: TLoc = getTemp(p, typ.returnType, needsInit=true)
        var list = initLoc(locCall, d.lode, OnUnknown)
        list.r = pl
        genAssignment(p, tmp, list, flags) # no need for deep copying
-        if canRaise: raiseExit(p)
+        if canRaise:
+          if not cleanupTemp(p, typ.returnType, tmp):
+            raiseExit(p)
        genAssignment(p, d, tmp, {})
  else:
    pl.add(");\n")
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -3239,7 +3239,8 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
      getNullValueAux(p, t, it, constOrNil, result, count, isConst, info)
  of nkRecCase:
    getNullValueAux(p, t, obj[0], constOrNil, result, count, isConst, info)
-    if count > 0: result.add ", "
+    var res = ""
+    if count > 0: res.add ", "
    var branch = Zero
    if constOrNil != nil:
      ## find kind value, default is zero if not specified
@@ -3253,18 +3254,21 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
          break

    let selectedBranch = caseObjDefaultBranch(obj, branch)
-    result.add "{"
+    res.add "{"
    var countB = 0
    let b = lastSon(obj[selectedBranch])
    # designated initilization is the only way to init non first element of unions
    # branches are allowed to have no members (b.len == 0), in this case they don't need initializer
    if b.kind == nkRecList and not isEmptyCaseObjectBranch(b):
-      result.add "._" & mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
-      getNullValueAux(p, t,  b, constOrNil, result, countB, isConst, info)
-      result.add "}"
+      res.add "._" & mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
+      getNullValueAux(p, t,  b, constOrNil, res, countB, isConst, info)
+      res.add "}"
    elif b.kind == nkSym:
-      result.add "." & mangleRecFieldName(p.module, b.sym) & " = "
-      getNullValueAux(p, t,  b, constOrNil, result, countB, isConst, info)
+      res.add "." & mangleRecFieldName(p.module, b.sym) & " = "
+      getNullValueAux(p, t,  b, constOrNil, res, countB, isConst, info)
+    else:
+      return
+    result.add res
    result.add "}"

  of nkSym:
--- a/compiler/ccgstmts.nim
+++ b/compiler/ccgstmts.nim
@@ -301,13 +301,13 @@ proc genCppParamsForCtor(p: BProc; call: PNode; didGenTemp: var bool): string =
      result.add genCppInitializer(p.module, p, call[i][0].sym.typ, didGenTemp)
    else:
      #We need to test for temp in globals, see: #23657
-      let param = 
+      let param =
        if typ[i].kind in {tyVar} and call[i].kind == nkHiddenAddr:
          call[i][0]
        else:
          call[i]
-      if param.kind != nkBracketExpr or param.typ.kind in 
-        {tyRef, tyPtr, tyUncheckedArray, tyArray, tyOpenArray, 
+      if param.kind != nkBracketExpr or param.typ.kind in
+        {tyRef, tyPtr, tyUncheckedArray, tyArray, tyOpenArray,
          tyVarargs, tySequence, tyString, tyCstring, tyTuple}:
        let tempLoc = initLocExprSingleUse(p, param)
        didGenTemp = didGenTemp or tempLoc.k == locTemp
@@ -755,6 +755,18 @@ proc raiseExit(p: BProc) =
      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
        [p.nestedTryStmts[^1].label])

+proc raiseExitCleanup(p: BProc, destroy: string) =
+  assert p.config.exc == excGoto
+  if nimErrorFlagDisabled notin p.flags:
+    p.flags.incl nimErrorFlagAccessed
+    if p.nestedTryStmts.len == 0:
+      p.flags.incl beforeRetNeeded
+      # easy case, simply goto 'ret':
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) {$1; goto BeforeRet_;}$n", [destroy])
+    else:
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) {$2; goto LA$1_;}$n",
+        [p.nestedTryStmts[^1].label, destroy])
+
 proc finallyActions(p: BProc) =
  if p.config.exc != excGoto and p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept:
    # if the current try stmt have a finally block,
@@ -784,10 +796,14 @@ proc genRaiseStmt(p: BProc, t: PNode) =
    var e = rdLoc(a)
    discard getTypeDesc(p.module, t[0].typ)
    var typ = skipTypes(t[0].typ, abstractPtrs)
-    # XXX For reasons that currently escape me, this is only required by the new
-    # C++ based exception handling:
-    if p.config.exc == excCpp:
+    case p.config.exc
+    of excCpp:
      blockLeaveActions(p, howManyTrys = 0, howManyExcepts = p.inExceptBlockLen)
+    of excGoto:
+      blockLeaveActions(p, howManyTrys = 0,
+        howManyExcepts = (if p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept: 1 else: 0))
+    else:
+      discard
    genLineDir(p, t)
    if isImportedException(typ, p.config):
      lineF(p, cpsStmts, "throw $1;$n", [e])
@@ -1566,14 +1582,14 @@ proc genAsmStmt(p: BProc, t: PNode) =
      if whichPragma(i) == wAsmSyntax:
        asmSyntax = i[1].strVal

-  if asmSyntax != "" and 
+  if asmSyntax != "" and
     not (
      asmSyntax == "gcc" and hasGnuAsm in CC[p.config.cCompiler].props or
      asmSyntax == "vcc" and hasGnuAsm notin CC[p.config.cCompiler].props):
    localError(
-      p.config, t.info, 
+      p.config, t.info,
      "Your compiler does not support the specified inline assembler")
-  
+
  genAsmOrEmitStmt(p, t, isAsmStmt=true, s)
  # see bug #2362, "top level asm statements" seem to be a mis-feature
  # but even if we don't do this, the example in #2362 cannot possibly
--- a/compiler/ccgtypes.nim
+++ b/compiler/ccgtypes.nim
@@ -277,9 +277,12 @@ proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool =
      if rettype.isImportedCppType or t.isImportedCppType or
          (typ.callConv == ccCDecl and conf.selectedGC in {gcArc, gcAtomicArc, gcOrc}):
        # prevents nrvo for cdecl procs; # bug #23401
-        return false
-      result = containsGarbageCollectedRef(t) or
-          (t.kind == tyObject and not isObjLackingTypeField(t))
+        result = false
+      else:
+        result = containsGarbageCollectedRef(t) or
+            (t.kind == tyObject and not isObjLackingTypeField(t)) or
+            (getSize(conf, rettype) == szUnknownSize and (t.sym == nil or sfImportc notin t.sym.flags))
+
    else: result = false

 const
--- a/compiler/ccgutils.nim
+++ b/compiler/ccgutils.nim
@@ -90,6 +90,9 @@ proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool =
    if s.typ.sym != nil and sfForward in s.typ.sym.flags:
      # forwarded objects are *always* passed by pointers for consistency!
      result = true
+    elif s.typ.kind == tySink and conf.selectedGC notin {gcArc, gcAtomicArc, gcOrc, gcHooks}:
+      # bug #23354:
+      result = false
    elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
      result = true           # requested anyway
    elif (tfFinal in pt.flags) and (pt[0] == nil):
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -752,6 +752,7 @@ proc intLiteral(i: BiggestInt; result: var Rope)
 proc genLiteral(p: BProc, n: PNode; result: var Rope)
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope; argsCounter: var int)
 proc raiseExit(p: BProc)
+proc raiseExitCleanup(p: BProc, destroy: string)

 proc initLocExpr(p: BProc, e: PNode, flags: TLocFlags = {}): TLoc =
  result = initLoc(locNone, e, OnUnknown, flags)
--- a/compiler/condsyms.nim
+++ b/compiler/condsyms.nim
@@ -166,3 +166,4 @@ proc initDefines*(symbols: StringTableRef) =
  defineSymbol("nimHasWarnStdPrefix")

  defineSymbol("nimHasVtables")
+  defineSymbol("nimHasJsNoLambdaLifting")
--- a/compiler/evaltempl.nim
+++ b/compiler/evaltempl.nim
@@ -56,6 +56,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
          #  internalAssert c.config, false
          idTablePut(c.mapping, s, x)
        if sfGenSym in s.flags:
+          # TODO: getIdent(c.ic, "`" & x.name.s & "`gensym" & $c.instID)
          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $c.instID),
            if c.instLines: actual.info else: templ.info)
        else:
--- a/compiler/ic/ic.nim
+++ b/compiler/ic/ic.nim
@@ -33,7 +33,7 @@ type
    HasDatInitProc
    HasModuleInitProc

-  PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file
+  PackedModuleReader* = object ## the parts of a PackedEncoder that are part of the .rod file
    definedSymbols: string
    moduleFlags: TSymFlags
    includes*: seq[(LitId, string)] # first entry is the module filename itself
@@ -59,8 +59,43 @@ type
    emittedTypeInfo*: seq[string]
    backendFlags*: set[ModuleBackendFlag]

-    syms*: seq[PackedSym]
-    types*: seq[PackedType]
+    syms*: OrderedTable[int32, PackedSym]
+    types*: OrderedTable[int32, PackedType]
+    strings*: BiTable[string] # we could share these between modules.
+    numbers*: BiTable[BiggestInt] # we also store floats in here so
+                                  # that we can assure that every bit is kept
+    man*: LineInfoManager
+
+    cfg: PackedConfig
+
+  PackedModuleWriter* = object ## the parts of a PackedEncoder that are part of the .rod file
+    definedSymbols: string
+    moduleFlags: TSymFlags
+    includes*: seq[(LitId, string)] # first entry is the module filename itself
+    imports: seq[LitId] # the modules this module depends on
+    toReplay*: PackedTree # pragmas and VM specific state to replay.
+    topLevel*: PackedTree  # top level statements
+    bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
+    #producedGenerics*: Table[GenericKey, SymId]
+    exports*: seq[(LitId, int32)]
+    hidden: seq[(LitId, int32)]
+    reexports: seq[(LitId, PackedItemId)]
+    compilerProcs*: seq[(LitId, int32)]
+    converters*, methods*, trmacros*, pureEnums*: seq[int32]
+
+    typeInstCache*: seq[(PackedItemId, PackedItemId)]
+    procInstCache*: seq[PackedInstantiation]
+    attachedOps*: seq[(PackedItemId, TTypeAttachedOp, PackedItemId)]
+    methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)]
+    enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
+    methodsPerType*: seq[(PackedItemId, PackedItemId)]
+    dispatchers*: seq[PackedItemId]
+
+    emittedTypeInfo*: seq[string]
+    backendFlags*: set[ModuleBackendFlag]
+
+    syms*: OrderedTable[int32, PackedSym]
+    types*: OrderedTable[int32, PackedType]
    strings*: BiTable[string] # we could share these between modules.
    numbers*: BiTable[BiggestInt] # we also store floats in here so
                                  # that we can assure that every bit is kept
@@ -69,7 +104,7 @@ type
    cfg: PackedConfig

  PackedEncoder* = object
-    #m*: PackedModule
+    #m*: PackedModuleWriter
    thisModule*: int32
    lastFile*: FileIndex # remember the last lookup entry.
    lastLit*: LitId
@@ -80,7 +115,7 @@ type
    symMarker*: IntSet #Table[ItemId, SymId]    # ItemId.item -> SymId
    config*: ConfigRef

-proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int;
+proc toString*(tree: PackedTree; pos: NodePos; m: PackedModuleWriter|PackedModuleReader; nesting: int;
               result: var string) =
  if result.len > 0 and result[^1] notin {' ', '\n'}:
    result.add ' '
@@ -116,11 +151,11 @@ proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int;
    result.add ")"
    #for i in 1..nesting*2: result.add ' '

-proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string =
+proc toString*(tree: PackedTree; n: NodePos; m: PackedModuleWriter|PackedModuleReader): string =
  result = ""
  toString(tree, n, m, 0, result)

-proc debug*(tree: PackedTree; m: PackedModule) =
+proc debug*(tree: PackedTree; m: PackedModuleWriter|PackedModuleReader) =
  stdout.write toString(tree, NodePos 0, m)

 proc isActive*(e: PackedEncoder): bool = e.config != nil
@@ -140,7 +175,7 @@ proc definedSymbolsAsString(config: ConfigRef): string =
    result.add ' '
    result.add d

-proc rememberConfig(c: var PackedEncoder; m: var PackedModule; config: ConfigRef; pc: PackedConfig) =
+proc rememberConfig(c: var PackedEncoder; m: var PackedModuleWriter; config: ConfigRef; pc: PackedConfig) =
  m.definedSymbols = definedSymbolsAsString(config)
  #template rem(x) =
  #  c.m.cfg.x = config.x
@@ -153,7 +188,7 @@ const
 when debugConfigDiff:
  import hashes, tables, intsets, sha1, strutils, sets

-proc configIdentical(m: PackedModule; config: ConfigRef): bool =
+proc configIdentical(m: PackedModuleReader; config: ConfigRef): bool =
  result = m.definedSymbols == definedSymbolsAsString(config)
  when debugConfigDiff:
    if not result:
@@ -183,7 +218,7 @@ proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string =
    result = $secureHashFile(fullpath)
    msgs.setHash(conf, fileIdx, result)

-proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId =
+proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModuleWriter): LitId =
  ## store a file index as a literal
  if x == c.lastFile:
    result = c.lastLit
@@ -197,16 +232,16 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId =
    c.lastLit = result
  assert result != LitId(0)

-proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
+proc toFileIndex*(x: LitId; m: PackedModuleReader; config: ConfigRef): FileIndex =
  result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x])

-proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
+proc includesIdentical(m: var PackedModuleReader; config: ConfigRef): bool =
  for it in mitems(m.includes):
    if hashFileCached(config, toFileIndex(it[0], m, config)) != it[1]:
      return false
  result = true

-proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) =
+proc initEncoder*(c: var PackedEncoder; m: var PackedModuleWriter; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) =
  ## setup a context for serializing to packed ast
  c.thisModule = moduleSym.itemId.module
  c.config = config
@@ -228,54 +263,54 @@ proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; co

  rememberConfig(c, m, config, pc)

-proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) =
+proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModuleWriter; f: FileIndex) =
  m.includes.add((toLitId(f, c, m), hashFileCached(c.config, f)))

-proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) =
+proc addImportFileDep*(c: var PackedEncoder; m: var PackedModuleWriter; f: FileIndex) =
  m.imports.add toLitId(f, c, m)

-proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+proc addHidden*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym) =
  assert s.kind != skUnknown
  let nameId = getOrIncl(m.strings, s.name.s)
  m.hidden.add((nameId, s.itemId.item))
  assert s.itemId.module == c.thisModule

-proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+proc addExported*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym) =
  assert s.kind != skUnknown
  assert s.itemId.module == c.thisModule
  let nameId = getOrIncl(m.strings, s.name.s)
  m.exports.add((nameId, s.itemId.item))

-proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+proc addConverter*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym) =
  assert c.thisModule == s.itemId.module
  m.converters.add(s.itemId.item)

-proc addTrmacro*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+proc addTrmacro*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym) =
  m.trmacros.add(s.itemId.item)

-proc addPureEnum*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+proc addPureEnum*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym) =
  assert s.kind == skType
  m.pureEnums.add(s.itemId.item)

-proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+proc addMethod*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym) =
  m.methods.add s.itemId.item

-proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+proc addReexport*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym) =
  assert s.kind != skUnknown
  if s.kind == skModule: return
  let nameId = getOrIncl(m.strings, s.name.s)
  m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m),
                                        item: s.itemId.item)))

-proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+proc addCompilerProc*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym) =
  let nameId = getOrIncl(m.strings, s.name.s)
  m.compilerProcs.add((nameId, s.itemId.item))

-proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule)
-proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
-proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId
+proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModuleWriter)
+proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModuleWriter): PackedItemId
+proc storeType(t: PType; c: var PackedEncoder; m: var PackedModuleWriter): PackedItemId

-proc flush(c: var PackedEncoder; m: var PackedModule) =
+proc flush(c: var PackedEncoder; m: var PackedModuleWriter) =
  ## serialize any pending types or symbols from the context
  while true:
    if c.pendingTypes.len > 0:
@@ -285,19 +320,19 @@ proc flush(c: var PackedEncoder; m: var PackedModule) =
    else:
      break

-proc toLitId(x: string; m: var PackedModule): LitId =
+proc toLitId(x: string; m: var PackedModuleWriter): LitId =
  ## store a string as a literal
  result = getOrIncl(m.strings, x)

-proc toLitId(x: BiggestInt; m: var PackedModule): LitId =
+proc toLitId(x: BiggestInt; m: var PackedModuleWriter): LitId =
  ## store an integer as a literal
  result = getOrIncl(m.numbers, x)

-proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo =
+proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModuleWriter): PackedLineInfo =
  pack(m.man, toLitId(x.fileIndex, c, m), x.line.int32, x.col.int32)
  #PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m))

-proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId {.inline.} =
+proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModuleWriter): PackedItemId {.inline.} =
  ## given a symbol, produce an ItemId with the correct properties
  ## for local or remote symbols, packing the symbol as necessary
  if s == nil or s.kind == skPackage:
@@ -331,7 +366,7 @@ template storeNode(dest, src, field) =
    nodeId = emptyNodeId
  dest.field = nodeId

-proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId =
+proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModuleWriter): PackedItemId =
  # We store multiple different trees in m.bodies. For this to work out, we
  # cannot immediately store types/syms. We enqueue them instead to ensure
  # we only write one tree into m.bodies after the other.
@@ -344,7 +379,7 @@ proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): Packed
    # the type belongs to this module, so serialize it here, eventually.
    addMissing(c, t)

-proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId =
+proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModuleWriter): PackedItemId =
  if s.isNil: return nilItemId
  assert s.itemId.module >= 0
  assert s.itemId.item >= 0
@@ -353,7 +388,7 @@ proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedIt
    # the sym belongs to this module, so serialize it here, eventually.
    addMissing(c, s)

-proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId =
+proc storeType(t: PType; c: var PackedEncoder; m: var PackedModuleWriter): PackedItemId =
  ## serialize a ptype
  if t.isNil: return nilItemId

@@ -362,10 +397,10 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
  result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)

  if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item):
-    if t.uniqueId.item >= m.types.len:
-      setLen m.types, t.uniqueId.item+1
+    #if t.uniqueId.item >= m.types.len:
+    #  setLen m.types, t.uniqueId.item+1

-    var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
+    var p = PackedType(id: t.uniqueId.item, kind: t.kind, flags: t.flags, callConv: t.callConv,
      size: t.size, align: t.align, nonUniqueId: t.itemId.item,
      paddingAtEnd: t.paddingAtEnd)
    storeNode(p, t, n)
@@ -380,7 +415,7 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
    # fill the reserved slot, nothing else:
    m.types[t.uniqueId.item] = p

-proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib =
+proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModuleWriter): PackedLib =
  ## the plib hangs off the psym via the .annex field
  if l.isNil: return
  result = PackedLib(kind: l.kind, generated: l.generated,
@@ -388,7 +423,7 @@ proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib
  )
  storeNode(result, l, path)

-proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId =
+proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModuleWriter): PackedItemId =
  ## serialize a psym
  if s.isNil: return nilItemId

@@ -396,12 +431,12 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
  result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)

  if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item):
-    if s.itemId.item >= m.syms.len:
-      setLen m.syms, s.itemId.item+1
+    #if s.itemId.item >= m.syms.len:
+    #  setLen m.syms, s.itemId.item+1

    assert sfForward notin s.flags

-    var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
+    var p = PackedSym(id: s.itemId.item, kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
      position: s.position, offset: s.offset, disamb: s.disamb, options: s.options,
      name: s.name.s.toLitId(m))

@@ -428,7 +463,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
    # fill the reserved slot, nothing else:
    m.syms[s.itemId.item] = p

-proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
+proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModuleWriter) =
  ## add a remote symbol reference to the tree
  let info = n.info.toPackedInfo(c, m)
  if n.typ != n.sym.typ:
@@ -443,7 +478,7 @@ proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pac
  ir.addNode(kind = nkNone, info = info,
             operand = n.sym.itemId.item)

-proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
+proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModuleWriter) =
  ## serialize a node into the tree
  if n == nil:
    ir.addNode(kind = nkNilRodNode, operand = 1, info = NoLineInfo)
@@ -491,13 +526,13 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
      toPackedNode(n[i], ir, c, m)
    ir.patch patchPos

-proc storeTypeInst*(c: var PackedEncoder; m: var PackedModule; s: PSym; inst: PType) =
+proc storeTypeInst*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym; inst: PType) =
  m.typeInstCache.add (storeSymLater(s, c, m), storeTypeLater(inst, c, m))

-proc addPragmaComputation*(c: var PackedEncoder; m: var PackedModule; n: PNode) =
+proc addPragmaComputation*(c: var PackedEncoder; m: var PackedModuleWriter; n: PNode) =
  toPackedNode(n, m.toReplay, c, m)

-proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
+proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModuleWriter) =
  let info = toPackedInfo(n.info, c, m)
  let patchPos = ir.prepare(n.kind, n.flags,
                            storeTypeLater(n.typ, c, m), info)
@@ -512,7 +547,7 @@ proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var
                 typeId = nilItemId, info = info)
  ir.patch patchPos

-proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) =
+proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModuleWriter) =
  case n.kind
  of routineDefs:
    toPackedProcDef(n, m.topLevel, encoder, m)
@@ -534,11 +569,11 @@ proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var Pac
  else:
    toPackedNode(n, m.topLevel, encoder, m)

-proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder; m: var PackedModule) =
+proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder; m: var PackedModuleWriter) =
  toPackedNodeIgnoreProcDefs(n, encoder, m)
  flush encoder, m

-proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var PackedModule) =
+proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var PackedModuleWriter) =
  ## Generic procs and generated `=hook`'s need explicit top-level entries so
  ## that the code generator can work without having to special case these. These
  ## entries will also be useful for other tools and are the cleanest design
@@ -548,7 +583,7 @@ proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var Packe
  #flush encoder, m

 proc storeAttachedProcDef*(t: PType; op: TTypeAttachedOp; s: PSym,
-                           encoder: var PackedEncoder; m: var PackedModule) =
+                           encoder: var PackedEncoder; m: var PackedModuleWriter) =
  assert s.kind in routineKinds
  assert isActive(encoder)
  let tid = storeTypeLater(t, encoder, m)
@@ -556,7 +591,7 @@ proc storeAttachedProcDef*(t: PType; op: TTypeAttachedOp; s: PSym,
  m.attachedOps.add (tid, op, sid)
  toPackedGeneratedProcDef(s, encoder, m)

-proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
+proc storeInstantiation*(c: var PackedEncoder; m: var PackedModuleWriter; s: PSym; i: PInstantiation) =
  var t = newSeq[PackedItemId](i.concreteTypes.len)
  for j in 0..high(i.concreteTypes):
    t[j] = storeTypeLater(i.concreteTypes[j], c, m)
@@ -565,7 +600,7 @@ proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i:
                                          concreteTypes: t)
  toPackedGeneratedProcDef(i.sym, c, m)

-proc storeExpansion*(c: var PackedEncoder; m: var PackedModule; info: TLineInfo; s: PSym) =
+proc storeExpansion*(c: var PackedEncoder; m: var PackedModuleWriter; info: TLineInfo; s: PSym) =
  toPackedNode(newSymNode(s, info), m.bodies, c, m)

 proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
@@ -596,7 +631,7 @@ when BenchIC:
 else:
  template bench(x, body) = body

-proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef;
+proc loadRodFile*(filename: AbsoluteFile; m: var PackedModuleReader; config: ConfigRef;
                  ignoreConfig = false): RodFileError =
  var f = rodfiles.open(filename.string)
  f.loadHeader()
@@ -613,6 +648,10 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
    f.loadSection section
    f.loadSeq data

+  template loadTableSection(section, data) {.dirty.} =
+    f.loadSection section
+    f.loadOrderedTable data
+
  template loadTabSection(section, data) {.dirty.} =
    f.loadSection section
    f.load data
@@ -645,8 +684,8 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
    loadTabSection topLevelSection, m.topLevel

    loadTabSection bodiesSection, m.bodies
-    loadSeqSection symsSection, m.syms
-    loadSeqSection typesSection, m.types
+    loadTableSection symsSection, m.syms
+    loadTableSection typesSection, m.types

    loadSeqSection typeInstCacheSection, m.typeInstCache
    loadSeqSection procInstCacheSection, m.procInstCache
@@ -672,7 +711,7 @@ proc storeError(err: RodFileError; filename: AbsoluteFile) =
  echo "Error: ", $err, "; couldn't write to ", filename.string
  removeFile(filename.string)

-proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var PackedModule) =
+proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var PackedModuleWriter) =
  flush encoder, m
  #rememberConfig(encoder, encoder.config)

@@ -691,6 +730,10 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
    f.storeSection section
    f.store data

+  template storeTableSection(section, data) {.dirty.} =
+    f.storeSection section
+    f.storeOrderedTable data
+
  storeTabSection stringsSection, m.strings

  storeSeqSection checkSumsSection, m.includes
@@ -714,9 +757,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
  storeTabSection topLevelSection, m.topLevel

  storeTabSection bodiesSection, m.bodies
-  storeSeqSection symsSection, m.syms
+  storeTableSection symsSection, m.syms

-  storeSeqSection typesSection, m.types
+  storeTableSection typesSection, m.types

  storeSeqSection typeInstCacheSection, m.typeInstCache
  storeSeqSection procInstCacheSection, m.procInstCache
@@ -740,7 +783,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac

  when false:
    # basic loader testing:
-    var m2: PackedModule
+    var m2: PackedModuleReader
    discard loadRodFile(filename, m2, encoder.config)
    echo "loaded ", filename.string

@@ -766,9 +809,10 @@ type
  LoadedModule* = object
    status*: ModuleStatus
    symsInit, typesInit, loadedButAliveSetChanged*: bool
-    fromDisk*: PackedModule
-    syms: seq[PSym] # indexed by itemId
-    types: seq[PType]
+    fromDisk*: PackedModuleReader
+    toDisk*: PackedModuleWriter
+    syms: OrderedTable[int32, PSym] # indexed by itemId
+    types: OrderedTable[int32, PType]
    module*: PSym # the one true module symbol.
    iface, ifaceHidden: Table[PIdent, seq[PackedItemId]]
      # PackedItemId so that it works with reexported symbols too
@@ -961,11 +1005,11 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s:
        loadToReplayNodes(g, c.config, c.cache, m, g[int m])

    assert g[si].status in {loaded, storing, stored}
-    if not g[si].symsInit:
-      g[si].symsInit = true
-      setLen g[si].syms, g[si].fromDisk.syms.len
+    #if not g[si].symsInit:
+    #  g[si].symsInit = true
+    #  setLen g[si].syms, g[si].fromDisk.syms.len

-    if g[si].syms[s.item] == nil:
+    if g[si].syms.getOrDefault(s.item) == nil:
      if g[si].fromDisk.syms[s.item].kind != skModule:
        result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item)
        # store it here early on, so that recursions work properly:
@@ -1012,11 +1056,11 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
    assert g[si].status in {loaded, storing, stored}
    assert t.item > 0

-    if not g[si].typesInit:
-      g[si].typesInit = true
-      setLen g[si].types, g[si].fromDisk.types.len
+    #if not g[si].typesInit:
+    #  g[si].typesInit = true
+    #  setLen g[si].types, g[si].fromDisk.types.len

-    if g[si].types[t.item] == nil:
+    if g[si].types.getOrDefault(t.item) == nil:
      result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item)
      # store it here early on, so that recursions work properly:
      g[si].types[t.item] = result
@@ -1155,10 +1199,7 @@ proc loadProcBody*(config: ConfigRef, cache: IdentCache;
 proc loadTypeFromId*(config: ConfigRef, cache: IdentCache;
                     g: var PackedModuleGraph; module: int; id: PackedItemId): PType =
  bench g.loadType:
-    if id.item < g[module].types.len:
-      result = g[module].types[id.item]
-    else:
-      result = nil
+    result = g[module].types.getOrDefault(id.item)
    if result == nil:
      var decoder = PackedDecoder(
        lastModule: int32(-1),
@@ -1171,10 +1212,7 @@ proc loadTypeFromId*(config: ConfigRef, cache: IdentCache;
 proc loadSymFromId*(config: ConfigRef, cache: IdentCache;
                    g: var PackedModuleGraph; module: int; id: PackedItemId): PSym =
  bench g.loadSym:
-    if id.item < g[module].syms.len:
-      result = g[module].syms[id.item]
-    else:
-      result = nil
+    result = g[module].syms.getOrDefault(id.item)
    if result == nil:
      var decoder = PackedDecoder(
        lastModule: int32(-1),
@@ -1190,21 +1228,8 @@ proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; confi
  else:
    ItemId(module: toFileIndex(id.module, g[thisModule].fromDisk, config).int32, item: id.item)

-proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) =
-  var bugs = 0
-  for i in 1 .. high(m.syms):
-    if m.syms[i].kind == skUnknown:
-      echo "EMPTY ID ", i, " module ", moduleId, " ", toFullPath(config, FileIndex(moduleId))
-      inc bugs
-  assert bugs == 0
-  when false:
-    var nones = 0
-    for i in 1 .. high(m.types):
-      inc nones, m.types[i].kind == tyNone
-    assert nones < 1
-
 proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
-                           moduleSym: PSym; m: PackedModule) =
+                           moduleSym: PSym; m: PackedModuleWriter) =
  # For now only used for heavy debugging. In the future we could use this to reduce the
  # compiler's memory consumption.
  let idx = moduleSym.position
@@ -1302,7 +1327,7 @@ proc searchForCompilerproc*(m: LoadedModule; name: string): int32 =
 # ------------------------- .rod file viewer ---------------------------------

 proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
-  var m: PackedModule = PackedModule()
+  var m: PackedModuleReader = PackedModuleReader()
  let err = loadRodFile(rodfile, m, config, ignoreConfig=true)
  if err != ok:
    config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err
--- a/compiler/ic/integrity.nim
+++ b/compiler/ic/integrity.nim
@@ -10,7 +10,7 @@
 ## Integrity checking for a set of .rod files.
 ## The set must cover a complete Nim project.

-import std/sets
+import std/[sets, tables]

 when defined(nimPreviewSlimSystem):
  import std/assertions
@@ -100,26 +100,26 @@ proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) =
 proc checkTree(c: var CheckedContext; t: PackedTree) =
  for p in allNodes(t): checkNode(c, t, p)

-proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) =
+proc checkLocalSymIds(c: var CheckedContext; m: PackedModuleReader; symIds: seq[int32]) =
  for symId in symIds:
    assert symId >= 0 and symId < m.syms.len, $symId & " " & $m.syms.len

-proc checkModule(c: var CheckedContext; m: PackedModule) =
+proc checkModule(c: var CheckedContext; m: PackedModuleReader) =
  # We check that:
  # - Every symbol references existing types and symbols.
  # - Every tree node references existing types and symbols.
-  for i in 0..high(m.syms):
-    checkLocalSym c, int32(i)
+  for _, v in pairs(m.syms):
+    checkLocalSym c, v.id

  checkTree c, m.toReplay
  checkTree c, m.topLevel

  for e in m.exports:
-    assert e[1] >= 0 and e[1] < m.syms.len
+    #assert e[1] >= 0 and e[1] < m.syms.len
    assert e[0] == m.syms[e[1]].name

  for e in m.compilerProcs:
-    assert e[1] >= 0 and e[1] < m.syms.len
+    #assert e[1] >= 0 and e[1] < m.syms.len
    assert e[0] == m.syms[e[1]].name

  checkLocalSymIds c, m, m.converters
--- a/compiler/ic/navigator.nim
+++ b/compiler/ic/navigator.nim
@@ -11,7 +11,7 @@
 ## IDE-like features. It uses the set of .rod files to accomplish
 ## its task. The set must cover a complete Nim project.

-import std/sets
+import std/[sets, tables]

 from std/os import nil
 from std/private/miscdollars import toLocation
--- a/compiler/ic/packed_ast.nim
+++ b/compiler/ic/packed_ast.nim
@@ -47,6 +47,7 @@ type
    path*: NodeId

  PackedSym* = object
+    id*: int32
    kind*: TSymKind
    name*: LitId
    typ*: PackedItemId
@@ -71,6 +72,7 @@ type
    instantiatedFrom*: PackedItemId

  PackedType* = object
+    id*: int32
    kind*: TTypeKind
    callConv*: TCallingConvention
    #nodekind*: TNodeKind
--- a/compiler/ic/rodfiles.nim
+++ b/compiler/ic/rodfiles.nim
@@ -19,6 +19,8 @@ from std/typetraits import supportsCopyMem
 when defined(nimPreviewSlimSystem):
  import std/[syncio, assertions]

+import std / tables
+
 ## Overview
 ## ========
 ## `RodFile` represents a Rod File (versioned binary format), and the
@@ -170,6 +172,18 @@ proc storeSeq*[T](f: var RodFile; s: seq[T]) =
    for i in 0..<s.len:
      storePrim(f, s[i])

+proc storeOrderedTable*[K, T](f: var RodFile; s: OrderedTable[K, T]) =
+  if f.err != ok: return
+  if s.len >= high(int32):
+    setError f, tooBig
+    return
+  var lenPrefix = int32(s.len)
+  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    setError f, ioFailure
+  else:
+    for _, v in s:
+      storePrim(f, v)
+
 proc loadPrim*(f: var RodFile; s: var string) =
  ## Read a string, the length was stored as a prefix
  if f.err != ok: return
@@ -211,6 +225,19 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
    for i in 0..<lenPrefix:
      loadPrim(f, s[i])

+proc loadOrderedTable*[K, T](f: var RodFile; s: var OrderedTable[K, T]) =
+  ## `T` must be compatible with `copyMem`, see `loadPrim`
+  if f.err != ok: return
+  var lenPrefix = int32(0)
+  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    setError f, ioFailure
+  else:
+    s = initOrderedTable[K, T](lenPrefix)
+    for i in 0..<lenPrefix:
+      var x = default T
+      loadPrim(f, x)
+      s[x.id] = x
+
 proc storeHeader*(f: var RodFile; cookie = defaultCookie) =
  ## stores the header which is described by `cookie`.
  if f.err != ok: return
--- a/compiler/injectdestructors.nim
+++ b/compiler/injectdestructors.nim
@@ -872,7 +872,8 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
      for i in 1..<n.len:
        if n[i].kind == nkExprColonExpr:
          let field = lookupFieldAgain(t, n[i][0].sym)
-          if field != nil and sfCursor in field.flags:
+          if field != nil and (sfCursor in field.flags or field.typ.kind in {tyOpenArray, tyVarargs}):
+            # don't sink fields with openarray types
            result[i][1] = p(n[i][1], c, s, normal)
          else:
            result[i][1] = p(n[i][1], c, s, m)
--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -111,13 +111,25 @@ type
    blocks: seq[TBlock]
    extraIndent: int
    previousFileName: string  # For frameInfo inside templates.
+    # legacy: generatedParamCopies and up fields are used for jsNoLambdaLifting
+    generatedParamCopies: IntSet
+    up: PProc     # up the call chain; required for closure support

 template config*(p: PProc): ConfigRef = p.module.config

 proc indentLine(p: PProc, r: Rope): Rope =
  var p = p
-  let ind = p.blocks.len + p.extraIndent
-  result = repeat(' ', ind*2) & r
+  if jsNoLambdaLifting in p.config.legacyFeatures:
+    var ind = 0
+    while true:
+      inc ind, p.blocks.len + p.extraIndent
+      if p.up == nil or p.up.prc != p.prc.owner:
+        break
+      p = p.up
+    result = repeat(' ', ind*2) & r
+  else:
+    let ind = p.blocks.len + p.extraIndent
+    result = repeat(' ', ind*2) & r

 template line(p: PProc, added: string) =
  p.body.add(indentLine(p, rope(added)))
@@ -1200,12 +1212,13 @@ proc genIf(p: PProc, n: PNode, r: var TCompRes) =
 proc generateHeader(p: PProc, prc: PSym): Rope =
  result = ""
  let typ = prc.typ
-  if typ.callConv == ccClosure:
-    # we treat Env as the `this` parameter of the function
-    # to keep it simple
-    let env = prc.ast[paramsPos].lastSon
-    assert env.kind == nkSym, "env is missing"
-    env.sym.loc.r = "this"
+  if jsNoLambdaLifting notin p.config.legacyFeatures:
+    if typ.callConv == ccClosure:
+      # we treat Env as the `this` parameter of the function
+      # to keep it simple
+      let env = prc.ast[paramsPos].lastSon
+      assert env.kind == nkSym, "env is missing"
+      env.sym.loc.r = "this"

  for i in 1..<typ.n.len:
    assert(typ.n[i].kind == nkSym)
@@ -1239,7 +1252,8 @@ const

 proc needsNoCopy(p: PProc; y: PNode): bool =
  return y.kind in nodeKindsNeedNoCopy or
-        ((mapType(y.typ) != etyBaseIndex) and
+        ((mapType(y.typ) != etyBaseIndex or
+          (jsNoLambdaLifting in p.config.legacyFeatures and y.kind == nkSym and y.sym.kind == skParam)) and
          (skipTypes(y.typ, abstractInst).kind in
            {tyRef, tyPtr, tyLent, tyVar, tyCstring, tyProc, tyOwned, tyOpenArray} + IntegralTypes))

@@ -1590,7 +1604,30 @@ proc attachProc(p: PProc; s: PSym) =

 proc genProcForSymIfNeeded(p: PProc, s: PSym) =
  if not p.g.generatedSyms.containsOrIncl(s.id):
-    attachProc(p, s)
+    if jsNoLambdaLifting in p.config.legacyFeatures:
+      let newp = genProc(p, s)
+      var owner = p
+      while owner != nil and owner.prc != s.owner:
+        owner = owner.up
+      if owner != nil: owner.locals.add(newp)
+      else: attachProc(p, newp, s)
+    else:
+      attachProc(p, s)
+
+proc genCopyForParamIfNeeded(p: PProc, n: PNode) =
+  let s = n.sym
+  if p.prc == s.owner or needsNoCopy(p, n):
+    return
+  var owner = p.up
+  while true:
+    if owner == nil:
+      internalError(p.config, n.info, "couldn't find the owner proc of the closed over param: " & s.name.s)
+    if owner.prc == s.owner:
+      if not owner.generatedParamCopies.containsOrIncl(s.id):
+        let copy = "$1 = nimCopy(null, $1, $2);$n" % [s.loc.r, genTypeInfo(p, s.typ)]
+        owner.locals.add(owner.indentLine(copy))
+      return
+    owner = owner.up

 proc genVarInit(p: PProc, v: PSym, n: PNode)

@@ -1602,6 +1639,8 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
      internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
    if sfCompileTime in s.flags:
      genVarInit(p, s, if s.astdef != nil: s.astdef else: newNodeI(nkEmpty, s.info))
+    if jsNoLambdaLifting in p.config.legacyFeatures and s.kind == skParam:
+      genCopyForParamIfNeeded(p, n)
    let k = mapType(p, s.typ)
    if k == etyBaseIndex:
      r.typ = etyBaseIndex
@@ -2688,6 +2727,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
  #if gVerbosity >= 3:
  #  echo "BEGIN generating code for: " & prc.name.s
  var p = newProc(oldProc.g, oldProc.module, prc.ast, prc.options)
+  p.up = oldProc
  var returnStmt: Rope = ""
  var resultAsgn: Rope = ""
  var name = mangleName(p.module, prc)
@@ -2911,14 +2951,17 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
    else:
      genCall(p, n, r)
  of nkClosure:
-    let tmp = getTemp(p)
-    var a: TCompRes = default(TCompRes)
-    var b: TCompRes = default(TCompRes)
-    gen(p, n[0], a)
-    gen(p, n[1], b)
-    lineF(p, "$1 = $2.bind($3); $1.ClP_0 = $2; $1.ClE_0 = $3;$n", [tmp, a.rdLoc, b.rdLoc])
-    r.res = tmp
-    r.kind = resVal
+    if jsNoLambdaLifting in p.config.legacyFeatures:
+      gen(p, n[0], r)
+    else:
+      let tmp = getTemp(p)
+      var a: TCompRes = default(TCompRes)
+      var b: TCompRes = default(TCompRes)
+      gen(p, n[0], a)
+      gen(p, n[1], b)
+      lineF(p, "$1 = $2.bind($3); $1.ClP_0 = $2; $1.ClE_0 = $3;$n", [tmp, a.rdLoc, b.rdLoc])
+      r.res = tmp
+      r.kind = resVal
  of nkCurly: genSetConstr(p, n, r)
  of nkBracket: genArrayConstr(p, n, r)
  of nkPar, nkTupleConstr: genTupleConstr(p, n, r)
--- a/compiler/lambdalifting.nim
+++ b/compiler/lambdalifting.nim
@@ -239,6 +239,11 @@ proc interestingIterVar(s: PSym): bool {.inline.} =
 template isIterator*(owner: PSym): bool =
  owner.kind == skIterator and owner.typ.callConv == ccClosure

+template liftingHarmful(conf: ConfigRef; owner: PSym): bool =
+  ## lambda lifting can be harmful for JS-like code generators.
+  let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
+  jsNoLambdaLifting in conf.legacyFeatures and conf.backend == backendJs and not isCompileTime
+
 proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; idgen: IdGenerator; owner: PSym) =
  if owner.kind != skMacro:
    createTypeBoundOps(g, nil, refType.elementType, info, idgen)
@@ -255,6 +260,7 @@ proc genCreateEnv(env: PNode): PNode =

 proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
  # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
+  if liftingHarmful(g.config, owner): return n
  let iter = n.sym
  assert iter.isIterator

@@ -879,7 +885,8 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
                  idgen: IdGenerator; flags: TransformFlags): PNode =
  let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro

-  if body.kind == nkEmpty or
+  if body.kind == nkEmpty or (jsNoLambdaLifting in g.config.legacyFeatures and
+      g.config.backend == backendJs and not isCompileTime) or
      (fn.skipGenericOwner.kind != skModule and force notin flags):

    # ignore forward declaration:
@@ -939,6 +946,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
          break
        ...
    """
+  if liftingHarmful(g.config, owner): return body
  if not (body.kind == nkForStmt and body[^2].kind in nkCallKinds):
    localError(g.config, body.info, "ignored invalid for loop")
    return body
--- a/compiler/liftdestructors.nim
+++ b/compiler/liftdestructors.nim
@@ -157,7 +157,7 @@ proc genWasMovedCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
  result.add(newSymNode(op))
  result.add genAddr(c, x)

-proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
+proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool, enforceWasMoved = false) =
  case n.kind
  of nkSym:
    if c.filterDiscriminator != nil: return
@@ -167,6 +167,8 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
        enforceDefaultOp:
      defaultOp(c, f.typ, body, x.dotField(f), b)
    else:
+      if enforceWasMoved:
+        body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x.dotField(f))
      fillBody(c, f.typ, body, x.dotField(f), b)
  of nkNilLit: discard
  of nkRecCase:
@@ -205,7 +207,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
      branch[^1] = newNodeI(nkStmtList, c.info)

      fillBodyObj(c, n[i].lastSon, branch[^1], x, y,
-                  enforceDefaultOp = localEnforceDefaultOp)
+                  enforceDefaultOp = localEnforceDefaultOp, enforceWasMoved = c.kind == attachedAsgn)
      if branch[^1].len == 0: inc emptyBranches
      caseStmt.add(branch)
    if emptyBranches != n.len-1:
@@ -216,7 +218,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
      fillBodyObj(c, n[0], body, x, y, enforceDefaultOp = false)
    c.filterDiscriminator = oldfilterDiscriminator
  of nkRecList:
-    for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp)
+    for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp, enforceWasMoved)
  else:
    illFormedAstLocal(n, c.g.config)

@@ -282,6 +284,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
    c.kind = attachedDestructor
    fillBodyObjTImpl(c, t, body, blob, y)
    c.kind = prevKind
+
  else:
    fillBodyObjTImpl(c, t, body, x, y)

--- a/compiler/modulegraphs.nim
+++ b/compiler/modulegraphs.nim
@@ -220,7 +220,7 @@ proc isCachedModule(g: ModuleGraph; module: int): bool {.inline.} =
 proc isCachedModule*(g: ModuleGraph; m: PSym): bool {.inline.} =
  isCachedModule(g, m.position)

-proc simulateCachedModule(g: ModuleGraph; moduleSym: PSym; m: PackedModule) =
+proc simulateCachedModule(g: ModuleGraph; moduleSym: PSym; m: PackedModuleWriter) =
  when false:
    echo "simulating ", moduleSym.name.s, " ", moduleSym.position
  simulateLoadedModule(g.packed, g.config, g.cache, moduleSym, m)
@@ -230,7 +230,7 @@ proc initEncoder*(g: ModuleGraph; module: PSym) =
  if id >= g.encoders.len:
    setLen g.encoders, id+1
  ic.initEncoder(g.encoders[id],
-    g.packed[id].fromDisk, module, g.config, g.startupPackedConfig)
+    g.packed[id].toDisk, module, g.config, g.startupPackedConfig)

 type
  ModuleIter* = object
@@ -351,8 +351,8 @@ proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttached
  if g.config.symbolFiles != disabledSf:
    assert module < g.encoders.len
    assert isActive(g.encoders[module])
-    toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk)
-    #storeAttachedProcDef(t, op, value, g.encoders[module], g.packed[module].fromDisk)
+    toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].toDisk)
+    #storeAttachedProcDef(t, op, value, g.encoders[module], g.packed[module].toDisk)

 iterator getDispatchers*(g: ModuleGraph): PSym =
  for i in g.dispatchers.mitems:
@@ -555,14 +555,14 @@ proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) =
  if g.config.symbolFiles != disabledSf:
    #assert g.encoders[m.int32].isActive
    assert g.packed[m.int32].status != stored
-    g.packed[m.int32].fromDisk.emittedTypeInfo.add ti
+    g.packed[m.int32].toDisk.emittedTypeInfo.add ti
    #echo "added typeinfo ", m.int32, " ", ti, " suspicious ", not g.encoders[m.int32].isActive

 proc rememberFlag*(g: ModuleGraph; m: PSym; flag: ModuleBackendFlag) =
  if g.config.symbolFiles != disabledSf:
    #assert g.encoders[m.int32].isActive
    assert g.packed[m.position].status != stored
-    g.packed[m.position].fromDisk.backendFlags.incl flag
+    g.packed[m.position].toDisk.backendFlags.incl flag

 proc closeRodFile*(g: ModuleGraph; m: PSym) =
  if g.config.symbolFiles in {readOnlySf, v2Sf}:
@@ -571,14 +571,14 @@ proc closeRodFile*(g: ModuleGraph; m: PSym) =
    # not depend on the hard disk contents!
    let mint = m.position
    saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))),
-                g.encoders[mint], g.packed[mint].fromDisk)
+                g.encoders[mint], g.packed[mint].toDisk)
    g.packed[mint].status = stored

  elif g.config.symbolFiles == stressTest:
    # debug code, but maybe a good idea for production? Could reduce the compiler's
    # memory consumption considerably at the cost of more loads from disk.
    let mint = m.position
-    simulateCachedModule(g, m, g.packed[mint].fromDisk)
+    simulateCachedModule(g, m, g.packed[mint].toDisk)
    g.packed[mint].status = loaded

 proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -246,6 +246,8 @@ type
    emitGenerics
      ## generics are emitted in the module that contains them.
      ## Useful for libraries that rely on local passC
+    jsNoLambdaLifting
+      ## Old transformation for closures in JS backend

  SymbolFilesOption* = enum
    disabledSf, writeOnlySf, readOnlySf, v2Sf, stressTest
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -222,66 +222,7 @@ proc shouldCheckCaseCovered(caseTyp: PType): bool =
  else:
    discard

-proc endsInNoReturn(n: PNode): bool =
-  ## check if expr ends the block like raising or call of noreturn procs do
-  result = false # assume it does return
-
-  template checkBranch(branch) =
-    if not endsInNoReturn(branch):
-      # proved a branch returns
-      return false
-
-  var it = n
-  # skip these beforehand, no special handling needed
-  while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0:
-    it = it.lastSon
-
-  case it.kind
-  of nkIfStmt:
-    var hasElse = false
-    for branch in it:
-      checkBranch:
-        if branch.len == 2:
-          branch[1]
-        elif branch.len == 1:
-          hasElse = true
-          branch[0]
-        else:
-          raiseAssert "Malformed `if` statement during endsInNoReturn"
-    # none of the branches returned
-    result = hasElse # Only truly a no-return when it's exhaustive
-  of nkCaseStmt:
-    let caseTyp = skipTypes(it[0].typ, abstractVar-{tyTypeDesc})
-    # semCase should already have checked for exhaustiveness in this case
-    # effectively the same as having an else
-    var hasElse = caseTyp.shouldCheckCaseCovered()
-
-    # actual noreturn checks
-    for i in 1 ..< it.len:
-      let branch = it[i]
-      checkBranch:
-        case branch.kind
-        of nkOfBranch:
-          branch[^1]
-        of nkElifBranch:
-          branch[1]
-        of nkElse:
-          hasElse = true
-          branch[0]
-        else:
-          raiseAssert "Malformed `case` statement in endsInNoReturn"
-    # Can only guarantee a noreturn if there is an else or it's exhaustive
-    result = hasElse
-  of nkTryStmt:
-    checkBranch(it[0])
-    for i in 1 ..< it.len:
-      let branch = it[i]
-      checkBranch(branch[^1])
-    # none of the branches returned
-    result = true
-  else:
-    result = it.kind in nkLastBlockStmts or
-      it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags
+proc endsInNoReturn(n: PNode): bool

 proc commonType*(c: PContext; x: PType, y: PNode): PType =
  # ignore exception raising branches in case/if expressions
@@ -313,6 +254,8 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
    result.owner = getCurrOwner(c)
  else:
    result = newSym(kind, considerQuotedIdent(c, n), c.idgen, getCurrOwner(c), n.info)
+    if find(result.name.s, '`') >= 0:
+      result.flags.incl sfWasGenSym
  #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule:
  #  incl(result.flags, sfGlobal)
  when defined(nimsuggest):
@@ -322,7 +265,7 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
                 allowed: TSymFlags): PSym
  # identifier with visibility
 proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
-                        allowed: TSymFlags): PSym
+                        allowed: TSymFlags, fromTopLevel = false): PSym

 proc typeAllowedCheck(c: PContext; info: TLineInfo; typ: PType; kind: TSymKind;
                      flags: TTypeAllowedFlags = {}) =
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -168,8 +168,6 @@ type
    inUncheckedAssignSection*: int
    importModuleLookup*: Table[int, seq[int]] # (module.ident.id, [module.id])
    skipTypes*: seq[PNode] # used to skip types between passes in type section. So far only used for inheritance, sets and generic bodies.
-    delayedEffects*: Table[ItemId, seq[PSym]]
-    delayedEffectsInverted*: Table[ItemId, ItemId]
  TBorrowState* = enum
    bsNone, bsReturnNotMatch, bsNoDistinct, bsGeneric, bsNotSupported, bsMatch

@@ -333,7 +331,7 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext =
    graph.packed[id].module = module
    initEncoder graph, module

-template packedRepr*(c): untyped = c.graph.packed[c.module.position].fromDisk
+template packedRepr*(c): untyped = c.graph.packed[c.module.position].toDisk
 template encoder*(c): untyped = c.graph.encoders[c.module.position]

 proc addIncludeFileDep*(c: PContext; f: FileIndex) =
--- a/compiler/semfold.nim
+++ b/compiler/semfold.nim
@@ -771,7 +771,8 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
  of nkCast:
    var a = getConstExpr(m, n[1], idgen, g)
    if a == nil: return
-    if n.typ != nil and n.typ.kind in NilableTypes:
+    if n.typ != nil and n.typ.kind in NilableTypes and
+        not (n.typ.kind == tyProc and a.typ.kind == tyProc):
      # we allow compile-time 'cast' for pointer types:
      result = a
      result.typ = n.typ
--- a/compiler/seminst.nim
+++ b/compiler/seminst.nim
@@ -56,8 +56,11 @@ iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TypeMapping): PS
      elif t.kind in {tyGenericParam, tyConcept}:
        localError(c.config, a.info, errCannotInstantiateX % q.name.s)
        t = errorType(c)
-      elif isUnresolvedStatic(t) and c.inGenericContext == 0 and
-          c.matchedConcept == nil:
+      elif isUnresolvedStatic(t) and (q.typ.kind == tyStatic or
+            (q.typ.kind == tyGenericParam and
+              q.typ.genericParamHasConstraints and
+              q.typ.genericConstraint.kind == tyStatic)) and
+          c.inGenericContext == 0 and c.matchedConcept == nil:
        # generic/concept type bodies will try to instantiate static values but
        # won't actually use them
        localError(c.config, a.info, errCannotInstantiateX % q.name.s)
@@ -396,6 +399,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
  for _, param in paramTypes(result.typ):
    entry.concreteTypes[i] = param
    inc i
+  #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " ", entry.concreteTypes.len
  if tfTriggersCompileTime in result.typ.flags:
    incl(result.flags, sfCompileTime)
  n[genericParamsPos] = c.graph.emptyNode
@@ -424,7 +428,9 @@ proc generateInstance(c: PContext, fn: PSym, pt: TypeMapping,
    if result.magic notin {mSlice, mTypeOf}:
      # 'toOpenArray' is special and it is allowed to return 'openArray':
      paramsTypeCheck(c, result.typ)
+    #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " <-- NEW PROC!", " ", entry.concreteTypes.len
  else:
+    #echo "INSTAN ", fn.name.s, " ", typeToString(result.typ), " <-- CACHED! ", typeToString(oldPrc.typ), " ", entry.concreteTypes.len
    result = oldPrc
  popProcCon(c)
  popInfoContext(c.config)
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -1024,14 +1024,7 @@ proc trackCall(tracked: PEffects; n: PNode) =
        else:
          if laxEffects notin tracked.c.config.legacyFeatures and a.kind == nkSym and
              a.sym.kind in routineKinds:
-            if tfTrackedProc in a.sym.typ.flags:
-              propagateEffects(tracked, n, a.sym)
-            else:
-              if a.sym.typ.itemId notin tracked.c.delayedEffects:
-                tracked.c.delayedEffects[a.sym.typ.itemId] = @[tracked.owner]
-              else:
-                tracked.c.delayedEffects[a.sym.typ.itemId].add tracked.owner
-              tracked.c.delayedEffectsInverted[tracked.owner.typ.itemId] = a.sym.typ.itemId
+            propagateEffects(tracked, n, a.sym)
      else:
        mergeRaises(tracked, effectList[exceptionEffects], n)
        mergeTags(tracked, effectList[tagEffects], n)
@@ -1735,14 +1728,6 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
  if strictNotNil in c.features and s.kind in {skProc, skFunc, skMethod, skConverter}:
    checkNil(s, body, g.config, c.idgen)

-  if s.typ.itemId notin c.delayedEffectsInverted:
-    s.typ.flags.incl tfTrackedProc
-
-  if s.typ.itemId in c.delayedEffects:
-    for sym in c.delayedEffects[s.typ.itemId]:
-      trackProc(c, sym, sym.ast[bodyPos])
-    # todo call track delayedEffects recursively
-
 proc trackStmt*(c: PContext; module: PSym; n: PNode, isTopLevel: bool) =
  case n.kind
  of {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef, nkFuncDef,
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -132,17 +132,140 @@ proc semExprBranchScope(c: PContext, n: PNode; expectedType: PType = nil): PNode
  closeScope(c)

 const
-  skipForDiscardable = {nkIfStmt, nkIfExpr, nkCaseStmt, nkOfBranch,
-    nkElse, nkStmtListExpr, nkTryStmt, nkFinally, nkExceptBranch,
+  skipForDiscardable = {nkStmtList, nkStmtListExpr,
+    nkOfBranch, nkElse, nkFinally, nkExceptBranch,
    nkElifBranch, nkElifExpr, nkElseExpr, nkBlockStmt, nkBlockExpr,
    nkHiddenStdConv, nkHiddenDeref}

 proc implicitlyDiscardable(n: PNode): bool =
-  var n = n
-  while n.kind in skipForDiscardable: n = n.lastSon
-  result = n.kind in nkLastBlockStmts or
-           (isCallExpr(n) and n[0].kind == nkSym and
-           sfDiscardable in n[0].sym.flags)
+  # same traversal as endsInNoReturn
+  template checkBranch(branch) =
+    if not implicitlyDiscardable(branch):
+      return false
+
+  var it = n
+  # skip these beforehand, no special handling needed
+  while it.kind in skipForDiscardable and it.len > 0:
+    it = it.lastSon
+
+  case it.kind
+  of nkIfExpr, nkIfStmt:
+    for branch in it:
+      checkBranch:
+        if branch.len == 2:
+          branch[1]
+        elif branch.len == 1:
+          branch[0]
+        else:
+          raiseAssert "Malformed `if` statement during implicitlyDiscardable"
+    # all branches are discardable
+    result = true
+  of nkCaseStmt:
+    for i in 1 ..< it.len:
+      let branch = it[i]
+      checkBranch:
+        case branch.kind
+        of nkOfBranch:
+          branch[^1]
+        of nkElifBranch:
+          branch[1]
+        of nkElse:
+          branch[0]
+        else:
+          raiseAssert "Malformed `case` statement in endsInNoReturn"
+    # all branches are discardable
+    result = true
+  of nkTryStmt:
+    checkBranch(it[0])
+    for i in 1 ..< it.len:
+      let branch = it[i]
+      if branch.kind != nkFinally:
+        checkBranch(branch[^1])
+    # all branches are discardable
+    result = true
+  of nkCallKinds:
+    result = it[0].kind == nkSym and {sfDiscardable, sfNoReturn} * it[0].sym.flags != {}
+  of nkLastBlockStmts:
+    result = true
+  else:
+    result = false
+
+proc endsInNoReturn(n: PNode, returningNode: var PNode): bool =
+  ## check if expr ends the block like raising or call of noreturn procs do
+  result = false # assume it does return
+
+  template checkBranch(branch) =
+    if not endsInNoReturn(branch, returningNode):
+      # proved a branch returns
+      return false
+
+  var it = n
+  # skip these beforehand, no special handling needed
+  while it.kind in skipForDiscardable and it.len > 0:
+    it = it.lastSon
+
+  case it.kind
+  of nkIfExpr, nkIfStmt:
+    var hasElse = false
+    for branch in it:
+      checkBranch:
+        if branch.len == 2:
+          branch[1]
+        elif branch.len == 1:
+          hasElse = true
+          branch[0]
+        else:
+          raiseAssert "Malformed `if` statement during endsInNoReturn"
+    # none of the branches returned
+    result = hasElse # Only truly a no-return when it's exhaustive
+  of nkCaseStmt:
+    let caseTyp = skipTypes(it[0].typ, abstractVar-{tyTypeDesc})
+    # semCase should already have checked for exhaustiveness in this case
+    # effectively the same as having an else
+    var hasElse = caseTyp.shouldCheckCaseCovered()
+
+    # actual noreturn checks
+    for i in 1 ..< it.len:
+      let branch = it[i]
+      checkBranch:
+        case branch.kind
+        of nkOfBranch:
+          branch[^1]
+        of nkElifBranch:
+          branch[1]
+        of nkElse:
+          hasElse = true
+          branch[0]
+        else:
+          raiseAssert "Malformed `case` statement in endsInNoReturn"
+    # Can only guarantee a noreturn if there is an else or it's exhaustive
+    result = hasElse
+  of nkTryStmt:
+    checkBranch(it[0])
+    var lastIndex = it.len - 1
+    if it[lastIndex].kind == nkFinally:
+      # if finally is noreturn, then the entire statement is noreturn
+      if endsInNoReturn(it[lastIndex][^1], returningNode):
+        return true
+      dec lastIndex
+    for i in 1 .. lastIndex:
+      let branch = it[i]
+      checkBranch(branch[^1])
+    # none of the branches returned
+    result = true
+  of nkLastBlockStmts:
+    result = true
+  of nkCallKinds:
+    result = it[0].kind == nkSym and sfNoReturn in it[0].sym.flags
+    if not result:
+      returningNode = it
+  else:
+    result = false
+    returningNode = it
+
+proc endsInNoReturn(n: PNode): bool =
+  var dummy: PNode = nil
+  result = endsInNoReturn(n, dummy)

 proc fixNilType(c: PContext; n: PNode) =
  if isAtom(n):
@@ -165,13 +288,9 @@ proc discardCheck(c: PContext, result: PNode, flags: TExprFlags) =
        localError(c.config, result.info, "expression has no type: " &
               renderTree(result, {renderNoComments}))
      else:
-        var n = result
-        while n.kind in skipForDiscardable:
-          if n.kind == nkTryStmt: n = n[0]
-          else: n = n.lastSon
-
        # Ignore noreturn procs since they don't have a type
-        if n.endsInNoReturn:
+        var n = result
+        if result.endsInNoReturn(n):
          return

        var s = "expression '" & $n & "' is of type '" &
@@ -361,7 +480,7 @@ proc identWithin(n: PNode, s: PIdent): bool =

 proc semIdentDef(c: PContext, n: PNode, kind: TSymKind, reportToNimsuggest = true): PSym =
  if isTopLevel(c):
-    result = semIdentWithPragma(c, kind, n, {sfExported})
+    result = semIdentWithPragma(c, kind, n, {sfExported}, fromTopLevel = true)
    incl(result.flags, sfGlobal)
    #if kind in {skVar, skLet}:
    #  echo "global variable here ", n.info, " ", result.name.s
--- a/compiler/semtypes.nim
+++ b/compiler/semtypes.nim
@@ -540,7 +540,7 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
    result = newSymG(kind, n, c)

 proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
-                        allowed: TSymFlags): PSym =
+                        allowed: TSymFlags, fromTopLevel = false): PSym =
  if n.kind == nkPragmaExpr:
    checkSonsLen(n, 2, c.config)
    result = semIdentVis(c, kind, n[0], allowed)
@@ -555,11 +555,15 @@ proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
    else: discard
  else:
    result = semIdentVis(c, kind, n, allowed)
+    let invalidPragmasForPush = if fromTopLevel and sfWasGenSym notin result.flags:
+      {}
+    else:
+      {wExportc, wExportCpp, wDynlib}
    case kind
    of skField: implicitPragmas(c, result, n.info, fieldPragmas)
-    of skVar:   implicitPragmas(c, result, n.info, varPragmas)
-    of skLet:   implicitPragmas(c, result, n.info, letPragmas)
-    of skConst: implicitPragmas(c, result, n.info, constPragmas)
+    of skVar:   implicitPragmas(c, result, n.info, varPragmas-invalidPragmasForPush)
+    of skLet:   implicitPragmas(c, result, n.info, letPragmas-invalidPragmasForPush)
+    of skConst: implicitPragmas(c, result, n.info, constPragmas-invalidPragmasForPush)
    else: discard

 proc checkForOverlap(c: PContext, t: PNode, currentEx, branchIndex: int) =
--- a/compiler/sigmatch.nim
+++ b/compiler/sigmatch.nim
@@ -2491,7 +2491,7 @@ proc arrayConstr(c: PContext, n: PNode): PType =
  result = newTypeS(tyArray, c)
  rawAddSon(result, makeRangeType(c, 0, 0, n.info))
  addSonSkipIntLit(result, skipTypes(n.typ,
-      {tyGenericInst, tyVar, tyLent, tyOrdinal}), c.idgen)
+      {tyVar, tyLent, tyOrdinal}), c.idgen)

 proc arrayConstr(c: PContext, info: TLineInfo): PType =
  result = newTypeS(tyArray, c)
--- a/compiler/transf.nim
+++ b/compiler/transf.nim
@@ -513,6 +513,7 @@ proc generateThunk(c: PTransf; prc: PNode, dest: PType): PNode =

  # we cannot generate a proper thunk here for GC-safety reasons
  # (see internal documentation):
+  if jsNoLambdaLifting in c.graph.config.legacyFeatures and c.graph.config.backend == backendJs: return prc
  result = newNodeIT(nkClosure, prc.info, dest)
  var conv = newNodeIT(nkHiddenSubConv, prc.info, dest)
  conv.add(newNodeI(nkEmpty, prc.info))
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -1316,9 +1316,17 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
    result = sameTypeOrNilAux(a.elementType, b.elementType, c) and
        sameValue(a.n[0], b.n[0]) and
        sameValue(a.n[1], b.n[1])
-  of tyGenericInst, tyAlias, tyInferred, tyIterable:
+  of tyAlias, tyInferred, tyIterable:
    cycleCheck()
    result = sameTypeAux(a.skipModifier, b.skipModifier, c)
+  of tyGenericInst:
+    # BUG #23445
+    # The type system must distinguish between `T[int] = object #[empty]#`
+    # and `T[float] = object #[empty]#`!
+    cycleCheck()
+    for ff, aa in underspecifiedPairs(a, b, 1, -1):
+      if not sameTypeAux(ff, aa, c): return false
+    result = sameTypeAux(a.skipModifier, b.skipModifier, c)
  of tyNone: result = false
  of tyConcept:
    result = exprStructuralEquivalent(a.n, b.n)
--- a/doc/manual.md
+++ b/doc/manual.md
@@ -8703,7 +8703,7 @@ after the last specified parameter. Nim string values will be converted to C
 strings automatically:

  ```Nim
-  proc printf(formatstr: cstring) {.nodecl, varargs.}
+  proc printf(formatstr: cstring) {.header: "<stdio.h>", varargs.}

  printf("hallo %s", "world") # "world" will be passed as C string
  ```
--- a/lib/nimbase.h
+++ b/lib/nimbase.h
@@ -272,11 +272,15 @@ __EMSCRIPTEN__
 #elif defined(__cplusplus)
 #define NIM_STATIC_ASSERT(x, msg) static_assert((x), msg)
 #else
-#define NIM_STATIC_ASSERT(x, msg) typedef int NIM_STATIC_ASSERT_AUX[(x) ? 1 : -1];
+#define _NIM_STATIC_ASSERT_FINAL(x, append_name) typedef int NIM_STATIC_ASSERT_AUX ## append_name[(x) ? 1 : -1];
+#define _NIM_STATIC_ASSERT_STAGE_3(x, line)      _NIM_STATIC_ASSERT_FINAL(x, _AT_LINE_##line)
+#define _NIM_STATIC_ASSERT_STAGE_2(x, line)      _NIM_STATIC_ASSERT_STAGE_3(x, line)
+#define NIM_STATIC_ASSERT(x, msg)                _NIM_STATIC_ASSERT_STAGE_2(x,__LINE__)
 // On failure, your C compiler will say something like:
-//   "error: 'NIM_STATIC_ASSERT_AUX' declared as an array with a negative size"
-// we could use a better fallback to also show line number, using:
-// http://www.pixelbeat.org/programming/gcc/static_assert.html
+//   "error: 'NIM_STATIC_ASSERT_AUX_AT_LINE_XXX' declared as an array with a negative size"
+// Adding the line number helps to avoid redefinitions which are not allowed in
+// old GCC versions, however the order of evaluation for __LINE__ is a little tricky,
+// hence all the helper macros. See https://stackoverflow.com/a/3385694 for more info.
 #endif

 /* C99 compiler? */
--- a/lib/pure/hashes.nim
+++ b/lib/pure/hashes.nim
@@ -379,6 +379,145 @@ proc hashVmImplChar(x: openArray[char], sPos, ePos: int): Hash =
 proc hashVmImplByte(x: openArray[byte], sPos, ePos: int): Hash =
  raiseAssert "implementation override in compiler/vmops.nim"

+const k0 = 0xc3a5c85c97cb3127u64 # Primes on (2^63, 2^64) for various uses
+const k1 = 0xb492b66fbe98f273u64
+const k2 = 0x9ae16a3b2f90404fu64
+
+proc load4e(s: openArray[byte], o=0): uint32 {.inline.} =
+  uint32(s[o + 3]) shl 24 or uint32(s[o + 2]) shl 16 or
+  uint32(s[o + 1]) shl  8 or uint32(s[o + 0])
+
+proc load8e(s: openArray[byte], o=0): uint64 {.inline.} =
+  uint64(s[o + 7]) shl 56 or uint64(s[o + 6]) shl 48 or
+  uint64(s[o + 5]) shl 40 or uint64(s[o + 4]) shl 32 or
+  uint64(s[o + 3]) shl 24 or uint64(s[o + 2]) shl 16 or
+  uint64(s[o + 1]) shl  8 or uint64(s[o + 0])
+
+proc load4(s: openArray[byte], o=0): uint32 {.inline.} =
+  when nimvm: result = load4e(s, o)
+  else:
+    when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof
+    else: result = load4e(s, o)
+
+proc load8(s: openArray[byte], o=0): uint64 {.inline.} =
+  when nimvm: result = load8e(s, o)
+  else:
+    when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof
+    else: result = load8e(s, o)
+
+proc lenU(s: openArray[byte]): uint64 {.inline.} = s.len.uint64
+
+proc shiftMix(v: uint64): uint64 {.inline.} = v xor (v shr 47)
+
+proc rotR(v: uint64; bits: cint): uint64 {.inline.} =
+  (v shr bits) or (v shl (64 - bits))
+
+proc len16(u: uint64; v: uint64; mul: uint64): uint64 {.inline.} =
+  var a = (u xor v)*mul
+  a = a xor (a shr 47)
+  var b = (v xor a)*mul
+  b = b xor (b shr 47)
+  b*mul
+
+proc len0_16(s: openArray[byte]): uint64 {.inline.} =
+  if s.len >= 8:
+    let mul = k2 + 2*s.lenU
+    let a   = load8(s) + k2
+    let b   = load8(s, s.len - 8)
+    let c   = rotR(b, 37)*mul + a
+    let d   = (rotR(a, 25) + b)*mul
+    len16 c, d, mul
+  elif s.len >= 4:
+    let mul = k2 + 2*s.lenU
+    let a   = load4(s).uint64
+    len16 s.lenU + (a shl 3), load4(s, s.len - 4), mul
+  elif s.len > 0:
+    let a = uint32(s[0])
+    let b = uint32(s[s.len shr 1])
+    let c = uint32(s[s.len - 1])
+    let y = a      + (b shl 8)
+    let z = s.lenU + (c shl 2)
+    shiftMix(y*k2 xor z*k0)*k2
+  else: k2      # s.len == 0
+
+proc len17_32(s: openArray[byte]): uint64 {.inline.} =
+  let mul = k2 + 2*s.lenU
+  let a = load8(s)*k1
+  let b = load8(s, 8)
+  let c = load8(s, s.len - 8)*mul
+  let d = load8(s, s.len - 16)*k2
+  len16 rotR(a + b, 43) + rotR(c, 30) + d, a + rotR(b + k2, 18) + c, mul
+
+proc len33_64(s: openArray[byte]): uint64 {.inline.} =
+  let mul = k2 + 2*s.lenU
+  let a = load8(s)*k2
+  let b = load8(s, 8)
+  let c = load8(s, s.len - 8)*mul
+  let d = load8(s, s.len - 16)*k2
+  let y = rotR(a + b, 43) + rotR(c, 30) + d
+  let z = len16(y, a + rotR(b + k2, 18) + c, mul)
+  let e = load8(s, 16)*mul
+  let f = load8(s, 24)
+  let g = (y + load8(s, s.len - 32))*mul
+  let h = (z + load8(s, s.len - 24))*mul
+  len16 rotR(e + f, 43) + rotR(g, 30) + h, e + rotR(f + a, 18) + g, mul
+
+type Pair = tuple[first, second: uint64]
+
+proc weakLen32withSeeds2(w, x, y, z, a, b: uint64): Pair {.inline.} =
+  var a = a + w
+  var b = rotR(b + a + z, 21)
+  let c = a
+  a += x
+  a += y
+  b += rotR(a, 44)
+  result[0] = a + z
+  result[1] = b + c
+
+proc weakLen32withSeeds(s: openArray[byte]; o: int; a,b: uint64): Pair {.inline.} =
+  weakLen32withSeeds2 load8(s, o     ), load8(s, o + 8),
+                      load8(s, o + 16), load8(s, o + 24), a, b
+
+proc hashFarm(s: openArray[byte]): uint64 {.inline.} =
+  if s.len <= 16: return len0_16(s)
+  if s.len <= 32: return len17_32(s)
+  if s.len <= 64: return len33_64(s)
+  const seed = 81u64 # not const to use input `h`
+  var
+    o = 0         # s[] ptr arith -> variable origin variable `o`
+    x = seed
+    y = seed*k1 + 113
+    z = shiftMix(y*k2 + 113)*k2
+    v, w: Pair
+  x = x*k2 + load8(s)
+  let eos    = ((s.len - 1) div 64)*64
+  let last64 = eos + ((s.len - 1) and 63) - 63
+  while true:
+    x = rotR(x + y + v[0] + load8(s, o+8), 37)*k1
+    y = rotR(y + v[1] + load8(s, o+48), 42)*k1
+    x = x xor w[1]
+    y += v[0] + load8(s, o+40)
+    z = rotR(z + w[0], 33)*k1
+    v = weakLen32withSeeds(s, o+0 , v[1]*k1, x + w[0])
+    w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16))
+    swap z, x
+    inc o, 64
+    if o == eos: break
+  let mul = k1 + ((z and 0xff) shl 1)
+  o = last64
+  w[0] += (s.lenU - 1) and 63
+  v[0] += w[0]
+  w[0] += v[0]
+  x = rotR(x + y + v[0] + load8(s, o+8), 37)*mul
+  y = rotR(y + v[1] + load8(s, o+48), 42)*mul
+  x = x xor w[1]*9
+  y += v[0]*9 + load8(s, o+40)
+  z = rotR(z + w[0], 33)*mul
+  v = weakLen32withSeeds(s, o+0 , v[1]*mul, x + w[0])
+  w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16))
+  swap z, x
+  len16 len16(v[0],w[0],mul) + shiftMix(y)*k0 + z, len16(v[1],w[1],mul) + x, mul
+
 proc hash*(x: string): Hash =
  ## Efficient hashing of strings.
  ##
@@ -388,10 +527,13 @@ proc hash*(x: string): Hash =
  runnableExamples:
    doAssert hash("abracadabra") != hash("AbracadabrA")

-  when nimvm:
-    result = hashVmImpl(x, 0, high(x))
+  when defined nimPreviewHashFarm: # Default switched -> `not nimStringHash2`
+    result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high)))
  else:
-    result = murmurHash(toOpenArrayByte(x, 0, high(x)))
+    when nimvm:
+      result = hashVmImpl(x, 0, high(x))
+    else:
+      result = murmurHash(toOpenArrayByte(x, 0, high(x)))

 proc hash*(x: cstring): Hash =
  ## Efficient hashing of null-terminated strings.
@@ -400,14 +542,21 @@ proc hash*(x: cstring): Hash =
    doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA")
    doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA")

-  when nimvm:
-    hashVmImpl(x, 0, high(x))
-  else:
-    when not defined(js):
-      murmurHash(toOpenArrayByte(x, 0, x.high))
-    else:
+  when defined nimPreviewHashFarm: # Default switched -> `not nimStringHash2`
+    when defined js:
      let xx = $x
-      murmurHash(toOpenArrayByte(xx, 0, high(xx)))
+      result = cast[Hash](hashFarm(toOpenArrayByte(xx, 0, xx.high)))
+    else:
+      result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high)))
+  else:
+    when nimvm:
+      hashVmImpl(x, 0, high(x))
+    else:
+      when not defined(js):
+        murmurHash(toOpenArrayByte(x, 0, x.high))
+      else:
+        let xx = $x
+        murmurHash(toOpenArrayByte(xx, 0, high(xx)))

 proc hash*(sBuf: string, sPos, ePos: int): Hash =
  ## Efficient hashing of a string buffer, from starting
@@ -418,7 +567,10 @@ proc hash*(sBuf: string, sPos, ePos: int): Hash =
    var a = "abracadabra"
    doAssert hash(a, 0, 3) == hash(a, 7, 10)

-  murmurHash(toOpenArrayByte(sBuf, sPos, ePos))
+  when defined nimPreviewHashFarm: # Default switched -> `not nimStringHash2`
+    result = cast[Hash](hashFarm(toOpenArrayByte(sBuf, sPos, ePos)))
+  else:
+    murmurHash(toOpenArrayByte(sBuf, sPos, ePos))

 proc hashIgnoreStyle*(x: string): Hash =
  ## Efficient hashing of strings; style is ignored.
@@ -553,12 +705,18 @@ proc hash*[A](x: openArray[A]): Hash =
  ## Efficient hashing of arrays and sequences.
  ## There must be a `hash` proc defined for the element type `A`.
  when A is byte:
-    result = murmurHash(x)
-  elif A is char:
-    when nimvm:
-      result = hashVmImplChar(x, 0, x.high)
+    when defined nimPreviewHashFarm: # Default switched -> `not nimStringHash2`
+      result = cast[Hash](hashFarm(x))
    else:
-      result = murmurHash(toOpenArrayByte(x, 0, x.high))
+      result = murmurHash(x)
+  elif A is char:
+    when defined nimPreviewHashFarm: # Default switched -> `not nimStringHash2`
+      result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high)))
+    else:
+      when nimvm:
+        result = hashVmImplChar(x, 0, x.high)
+      else:
+        result = murmurHash(toOpenArrayByte(x, 0, x.high))
  else:
    result = 0
    for a in x:
@@ -576,15 +734,21 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash =
    doAssert hash(a, 0, 1) == hash(a, 3, 4)

  when A is byte:
-    when nimvm:
-      result = hashVmImplByte(aBuf, sPos, ePos)
+    when defined nimPreviewHashFarm: # Default switched -> `not nimStringHash2`
+      result = cast[Hash](hashFarm(toOpenArray(aBuf, sPos, ePos)))
    else:
-      result = murmurHash(toOpenArray(aBuf, sPos, ePos))
+      when nimvm:
+        result = hashVmImplByte(aBuf, sPos, ePos)
+      else:
+        result = murmurHash(toOpenArray(aBuf, sPos, ePos))
  elif A is char:
-    when nimvm:
-      result = hashVmImplChar(aBuf, sPos, ePos)
+    when defined nimPreviewHashFarm: # Default switched -> `not nimStringHash2`
+      result = cast[Hash](hashFarm(toOpenArrayByte(aBuf, sPos, ePos)))
    else:
-      result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos))
+      when nimvm:
+        result = hashVmImplChar(aBuf, sPos, ePos)
+      else:
+        result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos))
  else:
    for i in sPos .. ePos:
      result = result !& hash(aBuf[i])
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -35,12 +35,12 @@ proc newEIO(msg: string): ref IOError =
  new(result)
  result.msg = msg

-proc setFileSize(fh: FileHandle, newFileSize = -1): OSErrorCode =
-  ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1.
-  ## Space is only allocated if that is cheaper than writing to the file.  This
-  ## routine returns the last OSErrorCode found rather than raising to support
-  ## old rollback/clean-up code style. [ Should maybe move to std/osfiles. ]
-  if newFileSize == -1:
+proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode =
+  ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1,
+  ## allocating | freeing space from the file system.  This routine returns the
+  ## last OSErrorCode found rather than raising to support old rollback/clean-up
+  ## code style. [ Should maybe move to std/osfiles. ]
+  if newFileSize < 0 or newFileSize == oldSize:
    return
  when defined(windows):
    var sizeHigh = int32(newFileSize shr 32)
@@ -51,14 +51,18 @@ proc setFileSize(fh: FileHandle, newFileSize = -1): OSErrorCode =
        setEndOfFile(fh) == 0:
      result = lastErr
  else:
-    var e: cint # posix_fallocate truncates up when needed.
-    when declared(posix_fallocate):
-      while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR):
-        discard
-    if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1:
-      result = osLastError() # fallback arguable; Most portable, but allows SEGV
-    elif e != 0:
-      result = osLastError()
+    if newFileSize > oldSize: # grow the file
+      var e: cint # posix_fallocate truncates up when needed.
+      when declared(posix_fallocate):
+        while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR):
+          discard
+      if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1:
+        result = osLastError() # fallback arguable; Most portable BUT allows SEGV
+      elif e != 0:
+        result = osLastError()
+    else: # shrink the file
+      if ftruncate(fh.cint, newFileSize) == -1:
+        result = osLastError()

 type
  MemFile* = object      ## represents a memory mapped file
@@ -255,41 +259,31 @@ proc open*(filename: string, mode: FileMode = fmRead,
      flags = flags or O_CREAT or O_TRUNC
      var permissionsMode = S_IRUSR or S_IWUSR
      result.handle = open(filename, flags, permissionsMode)
+      if result.handle != -1:
+        if (let e = setFileSize(result.handle.FileHandle, newFileSize);
+            e != 0.OSErrorCode): fail(e, "error setting file size")
    else:
      result.handle = open(filename, flags)

    if result.handle == -1:
-      # XXX: errno is supposed to be set here
-      # Is there an exception that wraps it?
      fail(osLastError(), "error opening file")

-    if (let e = setFileSize(result.handle.FileHandle, newFileSize);
-        e != 0.OSErrorCode): fail(e, "error setting file size")
-
-    if mappedSize != -1:
-      result.size = mappedSize
-    else:
-      var stat: Stat
+    if mappedSize != -1: #XXX Logic here differs from `when windows` branch ..
+      result.size = mappedSize #.. which always fstats&Uses min(mappedSize, st).
+    else: # if newFileSize!=-1: result.size=newFileSize # if trust setFileSize
+      var stat: Stat  #^^.. BUT some FSes (eg. Linux HugeTLBfs) round to 2MiB.
      if fstat(result.handle, stat) != -1:
-        # XXX: Hmm, this could be unsafe
-        # Why is mmap taking int anyway?
-        result.size = int(stat.st_size)
+        result.size = stat.st_size.int # int may be 32-bit-unsafe for 2..<4 GiB
      else:
        fail(osLastError(), "error getting file size")

    result.flags = if mapFlags == cint(-1): MAP_SHARED else: mapFlags
-    #Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set
+    # Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set
    if int(result.flags and MAP_PRIVATE) == 0:
      result.flags = result.flags or MAP_SHARED

-    result.mem = mmap(
-      nil,
-      result.size,
-      if readonly: PROT_READ else: PROT_READ or PROT_WRITE,
-      result.flags,
-      result.handle,
-      offset)
-
+    let pr = if readonly: PROT_READ else: PROT_READ or PROT_WRITE
+    result.mem = mmap(nil, result.size, pr, result.flags, result.handle, offset)
    if result.mem == cast[pointer](MAP_FAILED):
      fail(osLastError(), "file mapping failed")

@@ -353,7 +347,7 @@ proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} =
      raise newException(IOError,
                         "Cannot resize MemFile opened with allowRemap=false")
    if newFileSize != f.size:
-      if (let e = setFileSize(f.handle.FileHandle, newFileSize);
+      if (let e = setFileSize(f.handle.FileHandle, newFileSize, f.size);
          e != 0.OSErrorCode): raiseOSError(e)
    when defined(linux): #Maybe NetBSD, too?
      # On Linux this can be over 100 times faster than a munmap,mmap cycle.
--- a/lib/pure/options.nim
+++ b/lib/pure/options.nim
@@ -117,9 +117,10 @@ proc option*[T](val: sink T): Option[T] {.inline.} =
    assert option[Foo](nil).isNone
    assert option(42).isSome

-  result.val = val
-  when T isnot SomePointer:
-    result.has = true
+  when T is SomePointer:
+    result = Option[T](val: val)
+  else:
+    result = Option[T](has: true, val: val)

 proc some*[T](val: sink T): Option[T] {.inline.} =
  ## Returns an `Option` that has the value `val`.
@@ -136,10 +137,9 @@ proc some*[T](val: sink T): Option[T] {.inline.} =

  when T is SomePointer:
    assert not val.isNil
-    result.val = val
+    result = Option[T](val: val)
  else:
-    result.has = true
-    result.val = val
+    result = Option[T](has: true, val: val)

 proc none*(T: typedesc): Option[T] {.inline.} =
  ## Returns an `Option` for this type that has no value.
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -692,7 +692,10 @@ proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdT

 proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} =
  ## Sleeps `milsecs` milliseconds.
+  ## A negative `milsecs` causes sleep to return immediately.
  when defined(windows):
+    if milsecs < 0:
+      return  # fixes #23732
    winlean.sleep(int32(milsecs))
  else:
    var a, b: Timespec
--- a/lib/pure/parseutils.nim
+++ b/lib/pure/parseutils.nim
@@ -460,6 +460,8 @@ proc parseBiggestInt*(s: openArray[char], number: var BiggestInt): int {.
    var res: BiggestInt
    doAssert parseBiggestInt("9223372036854775807", res) == 19
    doAssert res == 9223372036854775807
+    doAssert parseBiggestInt("-2024_05_09", res) == 11
+    doAssert res == -20240509
  var res = BiggestInt(0)
  # use 'res' for exception safety (don't write to 'number' in case of an
  # overflow exception):
@@ -474,10 +476,8 @@ proc parseInt*(s: openArray[char], number: var int): int {.
  ## `ValueError` is raised if the parsed integer is out of the valid range.
  runnableExamples:
    var res: int
-    doAssert parseInt("2019", res, 0) == 4
-    doAssert res == 2019
-    doAssert parseInt("2019", res, 2) == 2
-    doAssert res == 19
+    doAssert parseInt("-2024_05_02", res) == 11
+    doAssert res == -20240502
  var res = BiggestInt(0)
  result = parseBiggestInt(s, res)
  when sizeof(int) <= 4:
@@ -992,6 +992,10 @@ proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {.noSid
    var res: BiggestInt
    doAssert parseBiggestInt("9223372036854775807", res, 0) == 19
    doAssert res == 9223372036854775807
+    doAssert parseBiggestInt("-2024_05_09", res) == 11
+    doAssert res == -20240509
+    doAssert parseBiggestInt("-2024_05_02", res, 7) == 4
+    doAssert res == 502
  parseBiggestInt(s.toOpenArray(start, s.high), number)

 proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raises: [ValueError].} =
@@ -1000,10 +1004,10 @@ proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raise
  ## `ValueError` is raised if the parsed integer is out of the valid range.
  runnableExamples:
    var res: int
-    doAssert parseInt("2019", res, 0) == 4
-    doAssert res == 2019
-    doAssert parseInt("2019", res, 2) == 2
-    doAssert res == 19
+    doAssert parseInt("-2024_05_02", res) == 11
+    doAssert res == -20240502
+    doAssert parseInt("-2024_05_02", res, 7) == 4
+    doAssert res == 502
  parseInt(s.toOpenArray(start, s.high), number)


--- a/lib/std/syncio.nim
+++ b/lib/std/syncio.nim
@@ -874,7 +874,7 @@ proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} =
  var f: File = nil
  if open(f, filename, fmWrite):
    try:
-      f.writeBuffer(unsafeAddr content[0], content.len)
+      discard f.writeBuffer(unsafeAddr content[0], content.len)
    finally:
      close(f)
  else:
--- a/lib/system/alloc.nim
+++ b/lib/system/alloc.nim
@@ -93,8 +93,6 @@ type
    freeList: ptr FreeCell
    free: int            # how many bytes remain
    acc: int             # accumulator for small object allocation
-    when defined(gcDestructors):
-      sharedFreeList: ptr FreeCell # make no attempt at avoiding false sharing for now for this object field
    data {.align: MemAlign.}: UncheckedArray[byte]      # start of usable memory

  BigChunk = object of BaseChunk # not necessarily > PageSize!
@@ -109,7 +107,9 @@ type
  MemRegion = object
    when not defined(gcDestructors):
      minLargeObj, maxLargeObj: int
-    freeSmallChunks: array[0..max(1,SmallChunkSize div MemAlign-1), PSmallChunk]
+    freeSmallChunks: array[0..max(1, SmallChunkSize div MemAlign-1), PSmallChunk]
+    when defined(gcDestructors):
+      sharedFreeLists: array[0..max(1, SmallChunkSize div MemAlign-1), ptr FreeCell]
    flBitmap: uint32
    slBitmap: array[RealFli, uint32]
    matrix: array[RealFli, array[MaxSli, PBigChunk]]
@@ -777,8 +777,10 @@ when defined(gcDestructors):
    sysAssert c.next == nil, "c.next pointer must be nil"
    atomicPrepend a.sharedFreeListBigChunks, c

-  proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell) {.inline.} =
-    atomicPrepend c.sharedFreeList, f
+  proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell; size: int) {.inline.} =
+    atomicPrepend c.owner.sharedFreeLists[size], f
+
+  const MaxSteps = 20

  proc compensateCounters(a: var MemRegion; c: PSmallChunk; size: int) =
    # rawDealloc did NOT do the usual:
@@ -788,30 +790,26 @@ when defined(gcDestructors):
    # we split the list in order to achieve bounded response times.
    var it = c.freeList
    var x = 0
-    var maxIters = 20 # make it time-bounded
    while it != nil:
-      if maxIters == 0:
-        let rest = it.next.loada
-        if rest != nil:
-          it.next.storea nil
-          addToSharedFreeList(c, rest)
-        break
      inc x, size
-      it = it.next.loada
-      dec maxIters
-    inc(c.free, x)
+      let chunk = cast[PSmallChunk](pageAddr(it))
+      inc(chunk.free, x)
+      it = it.next
    dec(a.occ, x)

  proc freeDeferredObjects(a: var MemRegion; root: PBigChunk) =
    var it = root
-    var maxIters = 20 # make it time-bounded
+    var maxIters = MaxSteps # make it time-bounded
    while true:
+      let rest = it.next.loada
+      it.next.storea nil
+      deallocBigChunk(a, cast[PBigChunk](it))
      if maxIters == 0:
-        let rest = it.next.loada
-        it.next.storea nil
-        addToSharedFreeListBigChunks(a, rest)
+        if rest != nil:
+          addToSharedFreeListBigChunks(a, rest)
+          sysAssert a.sharedFreeListBigChunks != nil, "re-enqueing failed"
        break
-      it = it.next.loada
+      it = rest
      dec maxIters
      if it == nil: break

@@ -835,8 +833,6 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
      sysAssert c.size == PageSize, "rawAlloc 3"
      c.size = size
      c.acc = size
-      when defined(gcDestructors):
-        c.sharedFreeList = nil
      c.free = SmallChunkSize - smallChunkOverhead() - size
      sysAssert c.owner == addr(a), "rawAlloc: No owner set!"
      c.next = nil
@@ -853,10 +849,11 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
      when defined(gcDestructors):
        if c.freeList == nil:
          when hasThreadSupport:
-            c.freeList = atomicExchangeN(addr c.sharedFreeList, nil, ATOMIC_RELAXED)
+            # Steal the entire list from `sharedFreeList`:
+            c.freeList = atomicExchangeN(addr a.sharedFreeLists[s], nil, ATOMIC_RELAXED)
          else:
-            c.freeList = c.sharedFreeList
-            c.sharedFreeList = nil
+            c.freeList = a.sharedFreeLists[s]
+            a.sharedFreeLists[s] = nil
          compensateCounters(a, c, size)
      if c.freeList == nil:
        sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize,
@@ -923,7 +920,7 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
  if isSmallChunk(c):
    # `p` is within a small chunk:
    var c = cast[PSmallChunk](c)
-    var s = c.size
+    let s = c.size
    #       ^ We might access thread foreign storage here.
    # The other thread cannot possibly free this block as it's still alive.
    var f = cast[ptr FreeCell](p)
@@ -957,7 +954,7 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
          freeBigChunk(a, cast[PBigChunk](c))
    else:
      when defined(gcDestructors):
-        addToSharedFreeList(c, f)
+        addToSharedFreeList(c, f, s div MemAlign)
    sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %%
               s == 0, "rawDealloc 2")
  else:
--- a/lib/system/orc.nim
+++ b/lib/system/orc.nim
@@ -146,7 +146,7 @@ proc unregisterCycle(s: Cell) =
  let idx = s.rootIdx-1
  when false:
    if idx >= roots.len or idx < 0:
-      cprintf("[Bug!] %ld\n", idx)
+      cprintf("[Bug!] %ld %ld\n", idx, roots.len)
      rawQuit 1
  roots.d[idx] = roots.d[roots.len-1]
  roots.d[idx][0].rootIdx = idx+1
@@ -303,6 +303,14 @@ proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) =
        t.setColor(colBlack)
        trace(t, desc, j)

+const
+  defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128
+
+when defined(nimStressOrc):
+  const rootsThreshold = 10 # broken with -d:nimStressOrc: 10 and for havlak iterations 1..8
+else:
+  var rootsThreshold {.threadvar.}: int
+
 proc collectCyclesBacon(j: var GcEnv; lowMark: int) =
  # pretty direct translation from
  # https://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon01Concurrent.pdf
@@ -341,22 +349,25 @@ proc collectCyclesBacon(j: var GcEnv; lowMark: int) =
    s.rootIdx = 0
    collectColor(s, roots.d[i][1], colToCollect, j)

+  # Bug #22927: `free` calls destructors which can append to `roots`.
+  # We protect against this here by setting `roots.len` to 0 and also
+  # setting the threshold so high that no cycle collection can be triggered
+  # until we are out of this critical section:
+  when not defined(nimStressOrc):
+    let oldThreshold = rootsThreshold
+    rootsThreshold = high(int)
+  roots.len = 0
+
  for i in 0 ..< j.toFree.len:
    when orcLeakDetector:
      writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1])
    free(j.toFree.d[i][0], j.toFree.d[i][1])

+  when not defined(nimStressOrc):
+    rootsThreshold = oldThreshold
+
  inc j.freed, j.toFree.len
  deinit j.toFree
-  #roots.len = 0
-
-const
-  defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128
-
-when defined(nimStressOrc):
-  const rootsThreshold = 10 # broken with -d:nimStressOrc: 10 and for havlak iterations 1..8
-else:
-  var rootsThreshold {.threadvar.}: int

 when defined(nimOrcStats):
  var freedCyclicObjects {.threadvar.}: int
@@ -396,7 +407,8 @@ proc collectCycles() =
    collectCyclesBacon(j, 0)

  deinit j.traceStack
-  deinit roots
+  if roots.len == 0:
+    deinit roots

  when not defined(nimStressOrc):
    # compute the threshold based on the previous history
--- a/nimsuggest/nimsuggest.nim
+++ b/nimsuggest/nimsuggest.nim
@@ -134,7 +134,8 @@ const
  #List of currently supported capabilities. So lang servers/ides can iterate over and check for what's enabled
  Capabilities = [
    "con", #current NimSuggest supports the `con` commmand
-    "exceptionInlayHints"
+    "exceptionInlayHints",
+    "unknownFile", #current NimSuggest can handle unknown files
  ]

 proc parseQuoted(cmd: string; outp: var string; start: int): int =
@@ -758,15 +759,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =

  if gMode != mstdin:
    conf.writelnHook = proc (msg: string) = discard
-  # Find Nim's prefix dir.
-  let binaryPath = findExe("nim")
-  if binaryPath == "":
-    raise newException(IOError,
-        "Cannot find Nim standard library: Nim compiler not in PATH")
-  conf.prefixDir = AbsoluteDir binaryPath.splitPath().head.parentDir()
-  if not dirExists(conf.prefixDir / RelativeDir"lib"):
-    conf.prefixDir = AbsoluteDir""
-
+  conf.prefixDir = conf.getPrefixDir()
  #msgs.writelnHook = proc (line: string) = log(line)
  myLog("START " & conf.projectFull.string)

@@ -1065,10 +1058,6 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile,
  var fileIndex: FileIndex

  if not (cmd in {ideRecompile, ideGlobalSymbols}):
-    if not fileInfoKnown(conf, file):
-      myLog fmt "{file} is unknown, returning no results"
-      return
-
    fileIndex = fileInfoIdx(conf, file)
    msgs.setDirtyFile(
      conf,
@@ -1344,13 +1333,7 @@ else:
      conf.writelnHook = proc (msg: string) = discard
    # Find Nim's prefix dir.
    if nimPath == "":
-      let binaryPath = findExe("nim")
-      if binaryPath == "":
-        raise newException(IOError,
-            "Cannot find Nim standard library: Nim compiler not in PATH")
-      conf.prefixDir = AbsoluteDir binaryPath.splitPath().head.parentDir()
-      if not dirExists(conf.prefixDir / RelativeDir"lib"):
-        conf.prefixDir = AbsoluteDir""
+      conf.prefixDir = conf.getPrefixDir()
    else:
      conf.prefixDir = AbsoluteDir nimPath

--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -63,7 +63,7 @@ pkg "criterion", allowFailure = true # needs testing binary
 pkg "datamancer"
 pkg "dashing", "nim c tests/functional.nim"
 pkg "delaunay"
-pkg "dnsclient"
+pkg "dnsclient", allowFailure = true # super fragile
 pkg "docopt"
 pkg "dotenv"
 # when defined(linux): pkg "drchaos"
@@ -144,7 +144,6 @@ pkg "polypbren"
 pkg "presto"
 pkg "prologue", "nimble tcompile"
 pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim"
-pkg "pylib"
 pkg "rbtree"
 pkg "react", "nimble example"
 pkg "regex", "nim c src/regex"
--- a/tests/alloc/tmembug.nim
+++ b/tests/alloc/tmembug.nim
@@ -0,0 +1,54 @@
+discard """
+  joinable: false
+"""
+
+import std / [atomics, strutils, sequtils]
+
+type
+  BackendMessage* = object
+    field*: seq[int]
+
+var
+  chan1: Channel[BackendMessage]
+  chan2: Channel[BackendMessage]
+
+chan1.open()
+chan2.open()
+
+proc routeMessage*(msg: BackendMessage) =
+  discard chan2.trySend(msg)
+
+var
+  recv: Thread[void]
+  stopToken: Atomic[bool]
+
+proc recvMsg() =
+  while not stopToken.load(moRelaxed):
+    let resp = chan1.tryRecv()
+    if resp.dataAvailable:
+      routeMessage(resp.msg)
+      echo "child consumes ", formatSize getOccupiedMem()
+
+createThread[void](recv, recvMsg)
+
+const MESSAGE_COUNT = 100
+
+proc main() =
+  let msg: BackendMessage = BackendMessage(field: (0..500).toSeq())
+  for j in 0..0: #100:
+    echo "New iteration"
+
+    for _ in 1..MESSAGE_COUNT:
+      chan1.send(msg)
+    echo "After sending"
+
+    var counter = 0
+    while counter < MESSAGE_COUNT:
+      let resp = recv(chan2)
+      counter.inc
+    echo "After receiving ", formatSize getOccupiedMem()
+
+  stopToken.store true, moRelaxed
+  joinThreads(recv)
+
+main()
--- a/tests/alloc/tmembug2.nim
+++ b/tests/alloc/tmembug2.nim
@@ -0,0 +1,58 @@
+discard """
+  disabled: "true"
+"""
+
+import std / [atomics, strutils, sequtils, isolation]
+
+import threading / channels
+
+type
+  BackendMessage* = object
+    field*: seq[int]
+
+const MESSAGE_COUNT = 100
+
+var
+  chan1 = newChan[BackendMessage](MESSAGE_COUNT*2)
+  chan2 = newChan[BackendMessage](MESSAGE_COUNT*2)
+
+#chan1.open()
+#chan2.open()
+
+proc routeMessage*(msg: BackendMessage) =
+  var m = isolate(msg)
+  discard chan2.trySend(m)
+
+var
+  thr: Thread[void]
+  stopToken: Atomic[bool]
+
+proc recvMsg() =
+  while not stopToken.load(moRelaxed):
+    var resp: BackendMessage
+    if chan1.tryRecv(resp):
+      #if resp.dataAvailable:
+      routeMessage(resp)
+      echo "child consumes ", formatSize getOccupiedMem()
+
+createThread[void](thr, recvMsg)
+
+proc main() =
+  let msg: BackendMessage = BackendMessage(field: (0..5).toSeq())
+  for j in 0..100:
+    echo "New iteration"
+
+    for _ in 1..MESSAGE_COUNT:
+      chan1.send(msg)
+    echo "After sending"
+
+    var counter = 0
+    while counter < MESSAGE_COUNT:
+      let resp = recv(chan2)
+      counter.inc
+    echo "After receiving ", formatSize getOccupiedMem()
+
+  stopToken.store true, moRelaxed
+  joinThreads(thr)
+
+main()
--- a/tests/arc/tcaseobj.nim
+++ b/tests/arc/tcaseobj.nim
@@ -338,3 +338,29 @@ block:
      doAssert ff.s == 12

    mainSync()
+
+import std/sequtils
+
+# bug #23690
+type
+  SomeObj* = object of RootObj
+
+  Item* = object
+    case kind*: 0..1
+    of 0:
+      a*: int
+      b*: SomeObj
+    of 1:
+      c*: string
+
+  ItemExt* = object
+    a*: Item
+    b*: string
+
+proc do1(x: int): seq[(string, Item)] =
+  result = @[("zero", Item(kind: 1, c: "first"))]
+
+proc do2(x: int, e: ItemExt): seq[(string, ItemExt)] =
+  do1(x).map(proc(v: (string, Item)): auto = (v[0], ItemExt(a: v[1], b: e.b)))
+
+doAssert $do2(0, ItemExt(a: Item(kind: 1, c: "second"), b: "third")) == """@[("zero", (a: (kind: 1, c: "first"), b: "third"))]"""
--- a/tests/destructor/tgotoexc_leak.nim
+++ b/tests/destructor/tgotoexc_leak.nim
@@ -0,0 +1,19 @@
+discard """
+  output: '''0
+true'''
+  cmd: "nim c --gc:arc $file"
+"""
+
+# bug #22398
+
+for i in 0 ..< 10_000:
+  try:
+    try:
+      raise newException(ValueError, "")
+    except CatchableError:
+      discard
+      raise newException(ValueError, "") # or raise getCurrentException(), just raise works ok
+  except ValueError:
+    discard
+echo getOccupiedMem()
+echo getCurrentException() == nil
--- a/tests/discard/t23677.nim
+++ b/tests/discard/t23677.nim
@@ -0,0 +1,12 @@
+discard """
+  errormsg: "expression '0' is of type 'int literal(0)' and has to be used (or discarded); start of expression here: t23677.nim(1, 1)"
+  line: 10
+  column: 3
+"""
+
+# issue #23677
+
+if true:
+  0
+else: 
+  raise newException(ValueError, "err") 
--- a/tests/discard/tdiscardable.nim
+++ b/tests/discard/tdiscardable.nim
@@ -5,6 +5,7 @@ tdiscardable
 1
 something defered
 something defered
+hi
 '''
 """

@@ -110,3 +111,32 @@ block:

  doAssertRaises(ValueError):
    doAssert foo() == 12
+
+block: # issue #10440
+  proc x(): int {.discardable.} = discard
+  try:
+    x()
+  finally:
+    echo "hi"
+
+import macros
+
+block: # issue #14665
+  macro test(): untyped =   
+    let b = @[1, 2, 3, 4]
+
+    result = nnkStmtList.newTree()
+    var i = 0
+    while i < b.len:
+      if false:
+        # this quote do is mandatory, removing it fixes the problem
+        result.add quote do:
+          let testtest = 5
+      else:
+        result.add quote do:
+          let test = 6
+        inc i
+        # removing this continue fixes the problem too
+        continue
+      inc i
+  test()
--- a/tests/discard/tfinallyerrmsg.nim
+++ b/tests/discard/tfinallyerrmsg.nim
@@ -0,0 +1,19 @@
+discard """
+  cmd: "nim check $file"
+"""
+
+block: # issue #19672
+  try:
+    10 #[tt.Error
+    ^ expression '10' is of type 'int literal(10)' and has to be used (or discarded); start of expression here: tfinallyerrmsg.nim(5, 1)]#
+  finally:
+    echo "Finally block"
+
+block: # issue #13871
+  template t(body: int) =
+    try:
+      body
+    finally:
+      echo "expression"
+  t: 2 #[tt.Error
+     ^ expression '2' is of type 'int literal(2)' and has to be used (or discarded)]#
--- a/tests/errmsgs/t22852.nim
+++ b/tests/errmsgs/t22852.nim
@@ -0,0 +1,9 @@
+discard """
+  exitcode: 1
+  outputsub: '''
+Error: unhandled exception: value out of range: -2 notin 0 .. 9223372036854775807 [RangeDefect]
+'''
+"""
+
+# bug #22852
+echo [0][2..^2]
--- a/tests/generics/tuninstantiatedgenericcalls.nim
+++ b/tests/generics/tuninstantiatedgenericcalls.nim
@@ -140,3 +140,8 @@ block: # issue #1771

  var a: Foo[range[0..2], float]
  doAssert test(a) == 0.0
+
+block: # issue #23730
+  proc test(M: static[int]): array[1 shl M, int] = discard
+  doAssert len(test(3)) == 8
+  doAssert len(test(5)) == 32
--- a/tests/js/tjsffi.nim
+++ b/tests/js/tjsffi.nim
@@ -1,4 +1,5 @@
 discard """
+matrix: "--legacy:jsnolambdalifting;"
 output: '''
 3
 2
--- a/tests/metatype/twrong_same_type.nim
+++ b/tests/metatype/twrong_same_type.nim
@@ -0,0 +1,28 @@
+# bug #23418
+
+template mapIt*(x: untyped): untyped =
+  type OutType {.gensym.} = typeof(x) #typeof(x, typeOfProc)
+  newSeq[OutType](5)
+
+type F[E] = object
+
+proc start(v: int): F[(ValueError,)] = discard
+proc stop(v: int): F[tuple[]] = discard
+
+assert $typeof(mapIt(start(9))) == "seq[F[(ValueError,)]]"
+assert $typeof(mapIt(stop(9))) == "seq[F[tuple[]]]"
+
+# bug #23445
+
+type F2[T; I: static int] = distinct int
+
+proc start2(v: int): F2[void, 22] = discard
+proc stop2(v: int): F2[void, 33] = discard
+
+var a = mapIt(start2(5))
+
+assert $type(a) == "seq[F2[system.void, 22]]", $type(a)
+
+var b = mapIt(stop2(5))
+
+assert $type(b) == "seq[F2[system.void, 33]]", $type(b)
--- a/tests/misc/tcast.nim
+++ b/tests/misc/tcast.nim
@@ -1,6 +1,7 @@
 discard """
  output: '''
 Hello World
+Hello World
 Hello World'''
  joinable: false
 """
@@ -8,7 +9,7 @@ type MyProc = proc() {.cdecl.}
 type MyProc2 = proc() {.nimcall.}
 type MyProc3 = proc() #{.closure.} is implicit

-proc testProc()  = echo "Hello World"
+proc testProc() {.exportc:"foo".} = echo "Hello World"

 template reject(x) = doAssert(not compiles(x))

@@ -23,6 +24,10 @@ proc callPointer(p: pointer) =
  ffunc0()
  ffunc1()

+    # bug #5901
+  proc foo() {.importc.}
+  (cast[proc(a: int) {.cdecl.}](foo))(5)
+
 callPointer(cast[pointer](testProc))

 reject: discard cast[enum](0)
--- a/tests/pragmas/tpush.nim
+++ b/tests/pragmas/tpush.nim
@@ -99,3 +99,28 @@ block: # bug #23019
  k(w)
  {.pop.}
  {.pop.}
+
+{.push exportC.}
+
+block:
+  proc foo11() =
+    const factor = [1, 2, 3, 4]
+    doAssert factor[0] == 1
+  proc foo21() =
+    const factor = [1, 2, 3, 4]
+    doAssert factor[0] == 1
+
+  foo11()
+  foo21()
+
+template foo31() =
+  let factor = [1, 2, 3, 4]
+  doAssert factor[0] == 1
+template foo41() =
+  let factor = [1, 2, 3, 4]
+  doAssert factor[0] == 1
+
+foo31()
+foo41()
+
+{.pop.}
--- a/tests/refc/tsinkbug.nim
+++ b/tests/refc/tsinkbug.nim
@@ -0,0 +1,17 @@
+discard """
+  matrix: "--gc:refc; --gc:arc"
+  output: '''
+Value is: 42
+Value is: 42'''
+"""
+
+type AnObject* = object of RootObj
+  value*: int
+
+proc mutate(a: sink AnObject) =
+  a.value = 1
+
+var obj = AnObject(value: 42)
+echo "Value is: ", obj.value
+mutate(obj)
+echo "Value is: ", obj.value
--- a/tests/stdlib/thashes.nim
+++ b/tests/stdlib/thashes.nim
@@ -47,19 +47,22 @@ block hashes:
      doAssert hashWangYi1(456) == -6421749900419628582

 block empty:
+  const emptyStrHash = # Hash=int=4B on js even w/--jsbigint64:on => cast[Hash]
+    when defined nimPreviewHashFarm: cast[Hash](-7286425919675154353i64)
+    else: 0
  var
    a = ""
    b = newSeq[char]()
    c = newSeq[int]()
    d = cstring""
    e = "abcd"
-  doAssert hash(a) == 0
-  doAssert hash(b) == 0
+  doAssert hash(a) == emptyStrHash
+  doAssert hash(b) == emptyStrHash
  doAssert hash(c) == 0
-  doAssert hash(d) == 0
+  doAssert hash(d) == emptyStrHash
  doAssert hashIgnoreCase(a) == 0
  doAssert hashIgnoreStyle(a) == 0
-  doAssert hash(e, 3, 2) == 0
+  doAssert hash(e, 3, 2) == emptyStrHash

 block sameButDifferent:
  doAssert hash("aa bb aaaa1234") == hash("aa bb aaaa1234", 0, 13)
@@ -93,7 +96,11 @@ block largeSize: # longer than 4 characters
 proc main() =
  doAssert hash(0.0) == hash(0)
  # bug #16061
-  doAssert hash(cstring"abracadabra") == 97309975
+  when defined nimPreviewHashFarm: # Default switched -> `not nimStringHash2`
+    # Hash=int=4B on js even w/--jsbigint64:on => cast[Hash]
+    doAssert hash(cstring"abracadabra") == cast[Hash](-1119910118870047694i64)
+  else:
+    doAssert hash(cstring"abracadabra") == 97309975
  doAssert hash(cstring"abracadabra") == hash("abracadabra")

  when sizeof(int) == 8 or defined(js):
--- a/tests/stdlib/tjson.nim
+++ b/tests/stdlib/tjson.nim
@@ -1,5 +1,5 @@
 discard """
-  matrix: "--mm:refc; --backend:cpp --mm:refc; --backend:js --jsbigint64:off; --backend:js --jsbigint64:on"
+  matrix: "; --backend:cpp; --backend:js --jsbigint64:off; --backend:js --jsbigint64:on"
 """


@@ -51,7 +51,7 @@ for i in 0 .. 10000:
  except:
    discard
 # memory diff should less than 4M
-doAssert(abs(getOccupiedMem() - startMemory) < 4 * 1024 * 1024) # todo fixme doesn;t work for ORC
+doAssert(abs(getOccupiedMem() - startMemory) < 4 * 1024 * 1024)


 # test `$`
--- a/tests/threads/tmembug.nim
+++ b/tests/threads/tmembug.nim
@@ -0,0 +1,51 @@
+
+import std / [atomics, strutils, sequtils]
+
+type
+  BackendMessage* = object
+    field*: seq[int]
+
+var
+  chan1: Channel[BackendMessage]
+  chan2: Channel[BackendMessage]
+
+chan1.open()
+chan2.open()
+
+proc routeMessage*(msg: BackendMessage) =
+  discard chan2.trySend(msg)
+
+var
+  recv: Thread[void]
+  stopToken: Atomic[bool]
+
+proc recvMsg() =
+  while not stopToken.load(moRelaxed):
+    let resp = chan1.tryRecv()
+    if resp.dataAvailable:
+      routeMessage(resp.msg)
+      echo "child consumes ", formatSize getOccupiedMem()
+
+createThread[void](recv, recvMsg)
+
+const MESSAGE_COUNT = 100
+
+proc main() =
+  let msg: BackendMessage = BackendMessage(field: (0..500).toSeq())
+  for j in 0..0: #100:
+    echo "New iteration"
+
+    for _ in 1..MESSAGE_COUNT:
+      chan1.send(msg)
+    echo "After sending"
+
+    var counter = 0
+    while counter < MESSAGE_COUNT:
+      let resp = recv(chan2)
+      counter.inc
+    echo "After receiving ", formatSize getOccupiedMem()
+
+  stopToken.store true, moRelaxed
+  joinThreads(recv)
+
+main()
--- a/tests/views/tviews1.nim
+++ b/tests/views/tviews1.nim
@@ -124,3 +124,13 @@ proc bug22597 = # bug #22597
  doAssert i == 1

 bug22597()
+
+block: # bug #20048
+  type
+    Test = object
+      tokens: openArray[string]
+
+  func init(Self: typedesc[Test], tokens: openArray[string]): Self = Self(tokens: tokens)
+
+  let data = Test.init(["123"])
+  doAssert @(data.tokens) == @["123"] 
--- a/tests/views/tviews2.nim
+++ b/tests/views/tviews2.nim
@@ -58,6 +58,18 @@ block: # bug #16671

    f()

+block: # bug #15746
+  type
+    Reader = object
+      data: openArray[char]
+      current: int
+
+  proc initReader(data: openArray[char], offset = 0): Reader =
+    result = Reader(data: data, current: offset)
+
+  let s = "\x01\x00\x00\x00"
+  doAssert initReader(s).data[0].int == 1
+
 block:
  proc foo(x: openArray[char]) =
    discard x
Author	SHA1	Message	Date
ringabout	78c5846225	split `PackedModule` into reader and writer	2024-06-19 22:10:00 +08:00
ringabout	646bd99d46	[backport] fixes #23711 ; C code contains backtick`gensym (#23716 ) fixes #23711	2024-06-19 08:33:38 +02:00
c-blake	e645120362	Add Farm Hash conditioned upon `nimPreviewHashFarm` as 64-bit `Hash` (#23735 ) Unlike present Nim this actually fills `Hash` for `string` & related. For the curious, note that `hashData` remains the aboriginal Nim string hasher & `import hashes {.all.}` allows simultaneous test/time of {orig, murmur, farm} on your favorite CPU & back end compiler. Update tests also conditioned upon `nimPreviewHashFarm` so they should pass either with or without that `define` on. In `--jsbigint=on` mode, only the lower 32-bits of `Hash` match nimvm & run-time values because `type Hash = int` and on JS int=int32, not int64 as for 64-bit Nim platforms. Due to the matching, `const` Table should match run-time `Table` on all platforms. To operate in `--jsbigint=off` mode is feasible but needs much "double precision mul/xor/ror/shr-arithmetic"-style work. That is distracting & also of questionable value since JS added BigInt in 2018, ringabout added Nim support for it in 2021 & `nimPreviewHashFarm` is unlikely to swap from an opt-in to an opt-out default before 2025..2026 which will have given a backward looking time window of 7..8 years for deployment platforms - reasonably generous. Add a changelog entry for 2.2.	2024-06-19 06:49:57 +02:00
ringabout	9d08d26e33	adds a define nimHasJsNoLambdaLifting so we can use it in the config for compatibility (#23736 )	2024-06-19 08:26:15 +08:00
Andreas Rumpf	3f1de49e26	IC: use tables instead of huge seqs because the compiler can create l… (#23737 ) …ots of dummy syms and types	2024-06-18 22:41:22 +02:00
lit	2a658c64d8	fixes #23732 , os.sleep(-1) now returns immediately (#23734 ) fixes #23732	2024-06-18 17:39:34 +02:00
ringabout	c58b6e8df8	disable dnsclient because it is fragile (#23728 ) ``` Unhandled exception: /home/runner/work/Nim/Nim/pkgstemp/dnsclient/tests/test1.nim(28, 3) `rr.strings == @["dnsclient.nim"]` [AssertionDefect] [FAILED] query TXT [OK] query MX [OK] query CNAME [OK] query SRV Error: execution of an external program failed: '/home/runner/work/Nim/Nim/pkgstemp/dnsclient/tests/test1' Tip: 2 messages have been suppressed, use --verbose to show them. tools.nim(36) doCmd Error: Execution failed with exit code 1 ... Command: /home/runner/work/Nim/Nim/bin/nim c --noNimblePath -d:NimblePkgVersion=0.3.4 --hints:off -r --path:. /home/runner/work/Nim/Nim/pkgstemp/dnsclient/tests/test1 ```	2024-06-18 19:43:46 +08:00
metagn	128090c593	ignore uninstantiated static on match to base type [backport:2.0] (#23731 ) fixes #23730 Since #23188 the compiler errors when matching a type variable to an uninstantiated static value. However sometimes an uninstantiated static value is given even when only a type match is being performed to the base type of the static type, in the given issue this case is: ```nim proc foo[T: SomeInteger](x: T): int = int(x) proc bar(x: static int): array[foo(x), int] = discard discard bar(123) ``` To deal with this issue we only error when matching against a type variable constrained to `static`. Not sure if the `q.typ.kind == tyGenericParam and q.typ.genericConstraint == tyStatic` check is necessary, the code above for deciding whether the variable becomes `skConst` doesn't use it.	2024-06-18 06:54:12 +02:00
fakuivan	33f5ce80d6	Fix NIM_STATIC_ASSERT_AUX being redefined on different lines (#23729 ) fixes #17247 This generates a new NIM_STATIC_ASSERT_AUX variable for each line that NIM_STATIC_ASSERT is called from. While this can solve all existing issues in the current code base, this method is not effective for multiple asserts on a single line.	2024-06-18 06:53:41 +02:00
ringabout	4867931af3	implement `legacy:jsNoLambdaLifting` for compatibility (#23727 )	2024-06-17 19:06:38 +02:00
ringabout	ae4b47c5bd	fixes #20048 ; fixes #15746 ; don't sink object fields if it's of openarray type (#23608 ) fixes #20048 fixes #15746	2024-06-15 16:07:49 +02:00
Tomohiro	de1f7188eb	Fix example code in Nim manual that cannot be compiled without error (#23722 )	2024-06-15 10:34:26 +08:00
ringabout	948bb38335	ref #20653 ; fixes chronos empty case branches (#23706 ) ref #20653 ```nim Error* = object case kind*: ErrorType of ErrorA: discard of ErrorB: discard ``` For an object variants without fields, it shouldn't generate empty brackets for default values since there are no fields at all in case branches.	2024-06-14 15:55:08 +02:00
Andreas Rumpf	5996b12355	fixes a long standing bug with varargs type inference [backport] (#23720 )	2024-06-14 09:16:39 +02:00
c-blake	8037bbe327	Fix non-exported `memfiles.setFileSize` to be able to shrink files on posix via `memfiles.resize` (#23717 ) Fix non-exported `setFileSize` to take optional `oldSize` to (on posix) shrink differently than it grows (`ftruncate` not `posix_fallocate`) since it makes sense to assume the higher address space has already been allocated there and include the old file size in the `proc resize` call. Also, do not even try `setFileSize` in the first place unless the `open` itself works by moving the call into the `if newFileSize != -1` branch. Just cosmetics, also improve some old 2011 comments, note a logic diff for callers using both `mappedSize` & `newFileSize` from windows branch in case someone wants to fix that & simplify code formatting a little.	2024-06-14 08:23:26 +02:00
ringabout	0b5a938f57	nrvo for embedded importc'ed types (#23708 )	2024-06-12 20:51:52 +02:00
Andreas Rumpf	3770236bee	fixes #22927 ; no test case extractable [backport] (#23707 )	2024-06-12 14:27:49 +02:00
lit	3915fdc372	fixes #23513 , parseutils.nim: parseInt's doc example. (#23561 ) fixes #23513 Also, the old `runnableExample` is just a copy of `proc parseInt(openArray[char], var int, int)` variant (in Line 1000). --------- Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com>	2024-06-12 10:13:38 +08:00
ringabout	262ff648aa	[backport] fixes #23690 ; SIGSEGV with object variants and RTTI (#23703 ) fixes #23690 ```nim dest.`:state` = src.`:state` var :tmp_553651276 = dest.e1.a `=wasMoved`(dest.e1.a) dest.e1.a.kind = src.e1.a.kind case dest.e1.a.kind of 0: dest.e1.a.a = src.e1.a.a of 1: `=copy`(dest.e1.a.c, src.e1.a.c) case :tmp_553651276.kind of 0: of 1: `=destroy`(:tmp_553651276.c) ``` `dest.e1.a.kind = src.e1.a.kind` changes the discrimant but it fails to clear the memory of `dest.e1.a`. Before using hooks for copying, we need to clear the dest, e.g. `=wasMoved(dest.e1.a.c)`. ```nim dest.`:state` = src.`:state` var :tmp_553651276 = dest.e1.a `=wasMoved`(dest.e1.a) dest.e1.a.kind = src.e1.a.kind case dest.e1.a.kind of 0: `=wasMoved`(dest.e1.a.a) dest.e1.a.a = src.e1.a.a `=wasMoved`(dest.e1.a.b) of 1: `=wasMoved`(dest.e1.a.c) `=copy`(dest.e1.a.c, src.e1.a.c) case :tmp_553651276.kind of 0: of 1: `=destroy`(:tmp_553651276.c) ```	2024-06-11 05:55:08 +02:00
Andreas Rumpf	8cbbe12ee4	fixes #22398 ; [backport] (#23704 )	2024-06-10 18:43:23 +02:00
Juan M Gómez	1cbcbd9269	Fixes #23695 : On Linux, "nimsuggest" crashes if Nim is installed in /usr/bin and the library in /usr/lib/nim (#23697 ) (Not tested)	2024-06-10 16:17:02 +02:00
Andreas Rumpf	56c95758b2	fixes #23445 ; fixes #23418 [backport] (#23699 )	2024-06-09 08:16:05 +02:00
Juan M Gómez	c7ee16182e	nimsuggest v3+ handles unknownFile (#23696 )	2024-06-08 13:02:48 +02:00
ringabout	09b5ed251e	remove pkg "pylib" (#23691 ) https://github.com/Yardanico/nimpylib is 404 now	2024-06-07 21:07:22 +08:00
Andreas Rumpf	7039b8b5bc	fixes #23354 ; [backport] (#23685 )	2024-06-07 09:01:30 +02:00
ringabout	8f5ae28fab	fixes #22672 ; Destructor not called for result when exception is thrown (#23267 ) fixes #22672	2024-06-06 11:51:41 +02:00
Andreas Rumpf	69d0b73d66	fixes #22510 (#23100 )	2024-06-06 00:52:01 +02:00
ringabout	87e56cabbb	make `std/options` compatible with strictdefs (#23675 )	2024-06-05 20:54:25 +02:00
ringabout	2d1533f34f	fixes #5901 #21211 ; don't fold cast function types because of gcc 14 (#23683 ) follow up https://github.com/nim-lang/Nim/pull/6265 fixes #5901 fixes #21211 It causes many problems with gcc14 if we fold the cast function types. Let's check what it will break	2024-06-05 20:54:00 +02:00
metagn	42e8472ca6	fix noreturn/implicit discard check logic (#23681 ) fixes #10440, fixes #13871, fixes #14665, fixes #19672, fixes #23677 The false positive in #23677 was caused by behavior in `implicitlyDiscardable` where only the last node of `if`/`case`/`try` etc expressions were considered, as in the final node of the final branch (in this case `else`). To fix this we use the same iteration in `implicitlyDiscardable` that we use in `endsInNoReturn`, with the difference that for an `if`/`case`/`try` statement to be implicitly discardable, all of its branches must be implicitly discardable. `noreturn` calls are also considered implicitly discardable for this reason, otherwise stuff like `if true: discardableCall() else: error()` doesn't compile. However `endsInNoReturn` also had bugs, one where `finally` was considered in noreturn checking when it shouldn't, another where only `nkIfStmt` was checked and not `nkIfExpr`, and the node given for the error message was bad. So `endsInNoReturn` now skips over `skipForDiscardable` which no longer contains `nkIfStmt`/`nkCaseStmt`/`nkTryStmt`, stores the first encountered returning node in a var parameter for the error message, and handles `finally` and `nkIfExpr`. Fixing #23677 already broke a line in `syncio` so some package code might be affected.	2024-06-05 20:53:05 +02:00