progress

compiler/ast.nim

@@ -560,6 +560,16 @@ proc nextSymId(x: IdGenerator): ItemId {.inline.} =
 
 proc nextTypeId*(x: IdGenerator): ItemId {.inline.} =
   assert(not x.sealed)
+  when not defined(nimKochBootstrap):
+    if x.backendMinted:
+      # Share the loader's per-module backend TYPE counter (seeded from the
+      # module's `(unusedid)`) so a freshly-minted backend type sits ABOVE every
+      # loaded type — never colliding with a frontend type's `toId` (the bug that
+      # crashed cgen's `getTypeDescAux` cycle check on `AsyncBufferRef`). Mirrors
+      # `nextSymId` (see ast2nif.nextBackendTypeItem).
+      let it = nextBackendTypeItem(program, x.module)
+      if it >= 0'i32:
+        return backendItemId(x.module, it)
   inc x.typeId
   result = if x.backendMinted: backendItemId(x.module, x.typeId)
            else: itemId(x.module, x.typeId)

compiler/ast2nif.nim

@@ -830,6 +830,13 @@ var reexpModTag = registerTag("reexpmod")
 var offerTag = registerTag("offer")
 var typeOfferTag = registerTag("toffer")
 var modulesrcTag = registerTag("modulesrc")
+# `(unusedid <int>)` — the module's first FREE itemId after the frontend
+# (`.s.bif`) or the lower stage (`.t.bif`). The backend seeds its per-module
+# sym/type counters here so freshly-minted backend ids (closure envs, RTTI
+# hooks, temps) start ABOVE every loaded id — no `toId` collision is possible
+# by construction (replaces relying on the `@bk` module-marker bit, which the
+# loader dropped on type USES). Mirrors NIF's `.unusedname` directive.
+var unusedIdTag = registerTag("unusedid")
 
 proc registerNifAstTags*() =
   ## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag`
@@ -1475,7 +1482,8 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
                                               concreteTypes: seq[PType];
                                               genericParamsCount: int]] = @[];
                      typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[];
-                     resolvedImportDeps: seq[FileIndex] = @[]) =
+                     resolvedImportDeps: seq[FileIndex] = @[];
+                     firstUnusedId: int32 = 0) =
   var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule)
   w.deps = newIcBuilder(64)
   var content = newIcBuilder(300)
@@ -1606,6 +1614,10 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
 
   var dest = newIcBuilder(600)
   createStmtList(dest, rootInfo)
+  # First child: the backend id seed (see `(unusedid)` / readUnusedId).
+  dest.addParLe unusedIdTag, NoLineInfo
+  dest.addIntLit firstUnusedId.int64
+  dest.addParRi()
   addAll(dest, w.deps)
   # do not write the (stmts .. ) wrapper:
   addStmtsBody(dest, content)
@@ -1695,7 +1707,9 @@ type
     buf: TokenBuf      # the WHOLE module, parsed eagerly (Step 2: replaces the
                        # lazy byte-offset stream entirely — symbol/type loading
                        # AND the body reader now cursor over this resident buffer)
-    symCounter: int32
+    symCounter: int32  # seeded from the file's `(unusedid)` so backend syms
+                       # start above every frontend/lowered id (no collision)
+    typeCounter: int32 # ditto for backend TYPES (closure envs etc.)
     index: Table[string, NifIndexEntry]  # name -> entry; `offset` is a TOKEN
                                          # position in `buf` (was a byte offset)
     suffix: string
@@ -1737,6 +1751,19 @@ proc nextBackendSymItem*(c: var DecodeContext; module: int32): int32 =
   inc p[]
   result = p[]
 
+proc nextBackendTypeItem*(c: var DecodeContext; module: int32): int32 =
+  ## TYPE analogue of `nextBackendSymItem`: the `lower`/`cg` stages mint fresh
+  ## backend TYPES (closure-env objects, ptr wrappers) whose itemId must not
+  ## collide with the module's loaded types. Drawn from the per-module
+  ## `typeCounter`, which `moduleId` seeds from the file's `(unusedid)` so the
+  ## first minted type sits ABOVE every frontend/lowered type item. Returns -1
+  ## if the module is not loaded (caller falls back to the idgen's own counter).
+  let fi = module.FileIndex
+  if not c.mods.hasKey(fi): return -1'i32
+  let p = addr c.mods[fi].typeCounter
+  inc p[]
+  result = p[]
+
 proc setMainModule*(c: var DecodeContext; fileIdx: FileIndex) =
   ## Records the module that is being compiled fresh so that re-exports of its
   ## own symbols by dependencies are not turned into duplicate stubs.
@@ -1810,6 +1837,29 @@ proc buildPosIndex(buf: var TokenBuf; suffix: string): Table[string, NifIndexEnt
     else:
       inc c
 
+proc readUnusedId(buf: var TokenBuf): int32 =
+  ## Find the module's `(unusedid <int>)` directive — emitted as the FIRST child
+  ## of the top-level `(stmts ...)` by writeNifModule/writeLoweredModule — and
+  ## return its value (the first free itemId). 0 if absent (older artifact: the
+  ## backend then falls back to its own un-seeded counter, i.e. pre-`unusedid`
+  ## behaviour).
+  result = 0'i32
+  if buf.len == 0: return
+  var c = buf.beginRead()
+  if c.kind != TagLit: return            # outermost (stmts ...)
+  inc c                                  # descend into stmts body
+  while c.hasMore:
+    if c.kind == TagLit:
+      if tagName(c.tags, c.cursorTagId) == "unusedid":
+        inc c                            # into the unusedid body
+        if c.hasMore and c.kind == IntLit:
+          result = int32 intVal(c)
+        return
+      else:
+        skip c                           # not it; skip this whole subtree
+    else:
+      inc c
+
 proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): FileIndex =
   var isKnownFile = false
   result = c.infos.config.registerNifSuffix(suffix, isKnownFile)
@@ -1838,7 +1888,12 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}):
         "whose NIF file hasn't been written yet."
     var m = bif.load(modFile)
     let index = buildPosIndex(m.buf, suffix)
-    c.mods[result] = NifModule(buf: ensureMove m.buf, index: index, suffix: suffix)
+    # Seed the backend id counters ABOVE every id the file already uses, so a
+    # freshly-minted backend sym/type (closure env, RTTI hook, temp) can never
+    # share a `toId` with a loaded one. See `readUnusedId` / `(unusedid)`.
+    let seed = readUnusedId(m.buf)
+    c.mods[result] = NifModule(buf: ensureMove m.buf, index: index, suffix: suffix,
+                               symCounter: seed, typeCounter: seed)
 
 proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry =
   let ii = addr c.mods[module].index
@@ -2820,6 +2875,10 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]
             let replayNode = loadNode(c, cur, suffix, localSyms)
             if replayNode != nil:
               result.topLevel.sons.add replayNode
+      elif tagIs(cur, "unusedid"):
+        # backend id seed — consumed eagerly by `moduleId`/`readUnusedId`; just
+        # skip past it here so the rest of the header still loads.
+        skip cur
       elif tagIs(cur, "repconverter"): loadLogOp(c, result.logOps, cur, ConverterEntry, attachedTrace, module)
       elif tagIs(cur, "repdestroy"):   loadLogOp(c, result.logOps, cur, HookEntry, attachedDestructor, module)
       elif tagIs(cur, "repwasmoved"):  loadLogOp(c, result.logOps, cur, HookEntry, attachedWasMoved, module)
@@ -3070,6 +3129,17 @@ proc writeLoweredModule*(c: var DecodeContext; config: ConfigRef;
 
   var dest = newIcBuilder(600)
   createStmtList(dest, rootInfo)
+  # Carry the seed FORWARD: the lower stage minted backend syms/types from the
+  # per-module counters (seeded out of the `.s.bif`'s `(unusedid)`), so they now
+  # hold the post-lower high-water mark. Record it so the `cg` stage — which
+  # loads THIS `.t.bif` and mints still more (RTTI hooks) — seeds above it too.
+  let lfi = FileIndex thisModule
+  let loweredSeed = if c.mods.hasKey(lfi):
+                      max(c.mods[lfi].symCounter, c.mods[lfi].typeCounter)
+                    else: 0'i32
+  dest.addParLe unusedIdTag, NoLineInfo
+  dest.addIntLit loweredSeed.int64
+  dest.addParRi()
   addAll(dest, w.deps)
   addStmtsBody(dest, content)
   dest.addParRi()

compiler/cgen.nim

@@ -1547,6 +1547,17 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
   for i in 1..<prc.typ.n.len:
     let param = prc.typ.n[i].sym
     if param.typ.isCompileTimeOnly: continue
+    if prc.typ.callConv == ccClosure and param.name.s == ":envP":
+      # The hidden closure-env param is materialised by `closureSetup`, never a
+      # normal C parameter (`genProcParams` omits it from the signature). In a
+      # from-source build it lives only in the routine's AST params and never in
+      # `typ.n`, so this loop never reaches it. Under IC `closureParams` leaks it
+      # into `typ.n`; for a LOADED closure it is already present at header time
+      # (`genProcParams` fills its loc), but for a RE-DERIVED closure
+      # (`wasLoaded == false`) `transformBody` appends it only AFTER
+      # `genProcHeader` ran, so its `loc.snippet` is still empty here. Skip it to
+      # match the from-source invariant — `closureSetup` assigns its local below.
+      continue
     assignParam(p, param, prc.typ.returnType)
   closureSetup(p, prc)
   genProcBody(p, procBody)

compiler/nifbackend.nim

@@ -157,7 +157,7 @@ proc signatureHasMetaType(t: PType; depth: int = 0): bool =
   for k in t.kids:
     if signatureHasMetaType(k, depth + 1): return true
 
-proc ownsRuntimeRoutine(s: PSym; modPos: int; forLowering = false): bool =
+proc ownsRuntimeRoutine(s: PSym; modPos: int): bool =
   ## A concrete, non-generic, runtime routine with a real body, OWNED by the
   ## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a
   ## routine called only from other modules is still emitted by somebody) and
@@ -180,20 +180,20 @@ proc ownsRuntimeRoutine(s: PSym; modPos: int; forLowering = false): bool =
   ## iterator, which is expanded at each call site) and must be emitted by its
   ## owner — else a cross-module `for` over it links to nothing.
   ##
-  ## `forLowering`: the `lower` stage must ALSO transform the generic INSTANCES
-  ## this module serializes into its `.t.bif` (each demander keeps its own copy,
-  ## `itemId.module == modPos`). The `.t.bif` is the authoritative backend
-  ## artifact — every routine `cg` emits must arrive with its lowered body baked
-  ## in, NEVER re-derived in `cg` (re-derivation on the partially-loaded backend
-  ## state is exactly what crashed `newSelector`). The `cg`/emit-everywhere path
-  ## keeps the `sfFromGeneric` exclusion (instances are still deduped by content
-  ## name at merge); only the body-producing `lower` pass relaxes it.
+  ## Generic INSTANCES (`sfFromGeneric`) are NEVER an owned runtime routine — not
+  ## in `cg` and not in the `lower` stage. They are demanded by the backend's
+  ## emit-everywhere path and deduped by `merge` (content C name); the frontend
+  ## materialises them through the `(offer)` mechanism. The `lower` stage must
+  ## not transform an instance: a not-fully-concrete instance (a closure factory
+  ## over a `static` param, or a `$`/`=` op instance whose body resolves only at
+  ## its further-specialised use sites) still carries unresolved overload choices
+  ## and crashes `transformBody` (empty-`namePos` lambda, nil-typed const-fold).
   s.itemId.module == modPos and
   (s.kind in {skProc, skFunc, skConverter, skMethod} or
    (s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and
   s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
   s.magic == mNone and
-  (forLowering or sfFromGeneric notin s.flags) and
+  sfFromGeneric notin s.flags and
   sfDispatcher notin s.flags and
   {sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
   s.typ != nil and not signatureHasMetaType(s.typ) and
@@ -485,7 +485,7 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
   # would emit two `=destroy`/`=copy` runs).
   var seenNested = initIntSet()
   for s in moduleSymbolStubs(ast.program, FileIndex modPos):
-    if ownsRuntimeRoutine(s, modPos, forLowering = true):
+    if ownsRuntimeRoutine(s, modPos):
       # REUSE path (`icReuseSemLowering` ON): a routine already transformed during
       # sem (CT eval / macro / VM transform) carries its lowered body in the
       # `.s.nif` slot (loaded into `transformedBody`) — don't re-transform it.

compiler/options.nim

@@ -29,7 +29,7 @@ const
 
   nimEnableCovariance* = defined(nimEnableCovariance)
 
-  icFormatVersion* = "23"
+  icFormatVersion* = "24"
     ## Version of the IC cache format (the sem-NIF module layout written by
     ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
     ## that layout changes: `commandIc` wipes a nimcache whose `ic.version`

compiler/pipelines.nim

@@ -313,9 +313,13 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
       # NIF `deps` section is complete (the backend closure walk needs it), and
       # reused below for the `.s.deps` sidecar (frontend graph re-derivation).
       let resolvedImportDeps = graph.importDeps.getOrDefault(module.position.FileIndex, @[])
+      # The frontend's highest used itemId (max of the sym and type counters):
+      # the backend seeds its id minting ABOVE this so closure envs / RTTI hooks
+      # never share a `toId` with a frontend sym/type. See ast2nif `(unusedid)`.
+      let firstUnusedId = max(idgen.symId, idgen.typeId)
       writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
                      replayActions, implDeps, reexportedModuleSyms(graph, module),
-                     genericOffers, typeOffers, resolvedImportDeps)
+                     genericOffers, typeOffers, resolvedImportDeps, firstUnusedId)
       # The module's REAL direct imports (incl. macro-generated) for `nim ic`'s
       # graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep.
       var semDepPaths: seq[string] = @[]