IC: three closure/environment fixes found by differential grinding

A closure's environment type — and the `=destroy`/`=copy`/`=sink` the compiler
lifts for it — is minted by the BACKEND and exists in no module's semmed NIF.
Grinding a corpus of closure programs against `nim c` as the oracle turned up
three ways the per-module backend gets those wrong. `tests/closure` is now green
under `--ic:on`, and `tests/iter` goes from 13 failures to 9.

* **The env hook is emitted by nobody.** `emitsBodyInThisModule` walks up to the
  outermost enclosing routine to pick the TU that emits a body. For the env
  `=destroy` of a generic closure iterator defined in one module and instantiated
  in another, that walk lands on the module of the ORIGINAL generic — a module
  that never sees the instance — so the routine was emitted nowhere
  (`undefined reference to eqdestroy__c485__…`). A backend-minted routine has no
  owning module NIF at all: it is written into the `.t.bif` of every module that
  references it, re-homed there with `@bk`. Every referencing TU emits it now and
  the merge stage keeps one, like any other content-addressed definition.

* **The `:up` link is stored without an increment.** `env.:up = enclosingEnv` has
  to go through `=copy` (with the cyclic incref) or the parent's refcount is one
  too low, and at teardown the two envs' mutually recursive `=destroy`s each
  believe they hold the last reference and recurse until the stack is gone — a
  SIGSEGV after the program's own output has already been printed
  (`tests/iter/tnestedclosures`, "Test 3"). Whether it becomes a `=copy` depends
  on the up-field type's hooks existing at injection time. Whole-program cgen got
  that for free: a LATER lifting pass creates them and it runs before any
  routine's injection. The per-module backend injects a routine right after
  lifting it (`lower`), long before the module's top level is transformed at all
  (`cg`) — so both up-field assignment sites create the ops themselves.

* **Two backend hooks collide on one C name.** A backend-minted symbol is named
  `_c<itemId.item>`, a PER-PROCESS counter — fine while "nifc lifts, emits and
  compiles them in one run", which is what `mangleProcNameExt` assumed. But
  `lower` mints the env hooks of nested routines and `cg` mints those of the
  module's top level, and both land in the same translation unit: two unrelated
  `=destroy`s became one C function. `mangleProcNameExt` already makes an
  exception for hooks whose `disamb` is content-derived (`HookDisambBit`);
  `toNifSymName` now mirrors it, so the content-derived value survives the round
  trip instead of being overwritten by the loader.

`InstanceDisambBit`/`HookDisambBit` move to `astdef` — `ast2nif` names symbols by
them and cannot import `modulegraphs`.

Two new metamorphic tests, both of which fail on the previous compiler:
`tclosure_hooks` (generic closure iterator instantiated across modules) and
`tclosure_nested_iter` (closure iterator nested in a closure iterator).

`icFormatVersion` 37 -> 38 for the hook naming. `koch bootic` reaches its
byte-identical fixed point; `tests/closure`, `tests/destructor` (3 known
`--newruntime` failures), `tests/cpp` and the classic categories are unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

compiler/ast2nif.nim

@@ -329,7 +329,18 @@ proc toNifSymName(w: var Writer; sym: PSym): string =
     # copies this back into `disamb` (sn.count), so `globalName` round-trips.
     result = sym.name.s
     result.add '.'
-    result.addInt sym.itemId.item
+    if (sym.disamb and HookDisambBit) != 0'i32:
+      # EXCEPTION, mirroring `mangleProcNameExt`: a lifted hook's `disamb` is
+      # CONTENT-derived (`setHookDisamb`), so it is the same in every process,
+      # whereas `itemId.item` is a per-process backend counter. Such a hook DOES
+      # cross process boundaries — the `lower` stage mints the env hooks of
+      # nested routines while `cg` mints those of the module's top level, and
+      # both end up in the same translation unit — so two unrelated hooks
+      # collided on `_c<item>` and the merge stage kept one body for both
+      # (C accepted the mistyped call, C++ rejected it).
+      result.addInt sym.disamb
+    else:
+      result.addInt sym.itemId.item
     result.add '.'
     result.add modname(w.currentModule, w.infos.config)
     result.add BackendLocalMarker

compiler/astdef.nim

@@ -1046,6 +1046,20 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode =
 # handling for IC, they end up in IC indexes etc. Thus we "log" them in the module graph
 # and to pass them around to the NIF writer. This is not very elegant but it works.
 
+const
+  InstanceDisambBit* = 0x4000_0000'i32
+    ## Set in the `disamb` of routine instances whose value is content-derived
+    ## (see `modulegraphs.setInstanceDisamb`); keeps them disjoint from the
+    ## small counter range ordinary symbols draw from, so the NIF name
+    ## `name.disamb.module` stays collision-free within a module.
+  HookDisambBit* = 0x2000_0000'i32
+    ## Set in the `disamb` of synthesized type-bound operators and `$enum`
+    ## procs whose value is content-derived (see `modulegraphs.setHookDisamb`);
+    ## disjoint from both the small counter range and `InstanceDisambBit`.
+    ##
+    ## Both live here rather than in `modulegraphs` because `ast2nif` — which
+    ## cannot import that module — names symbols by them.
+
 type
   LogEntryKind* = enum
     HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry,

compiler/cgen.nim

@@ -150,6 +150,17 @@ proc emitsBodyInThisModule(m: BModule, prc: PSym): bool =
   # on demand and emits it nowhere → undefined at link.
   if (prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32:
     return true
+  # A BACKEND-MINTED routine (a hook or nested proc that lambda-lifting /
+  # `injectDestructorCalls` created during the `lower` stage) exists in no
+  # module's semmed NIF: it is written into the `.t.bif` of every module that
+  # references it, re-homed there with the `@bk` marker. So there IS no other
+  # `cg` process that could emit it, and the owner walk below — which lands on
+  # the ORIGINAL generic's module for the env `=destroy` of a generic closure
+  # iterator instantiated elsewhere — leaves it in no TU at all
+  # (`undefined reference to eqdestroy__c485__…`). Every referencing TU emits it;
+  # the merge stage keeps one.
+  if isBackendMinted(prc.itemId):
+    return true
   var top = prc
   while top.skipGenericOwner != nil and top.skipGenericOwner.kind != skModule:
     top = top.skipGenericOwner

compiler/lambdalifting.nim

@@ -675,6 +675,17 @@ proc rawClosureCreation(owner: PSym;
     if up != nil and upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) == up.typ.skipTypes({tyOwned, tyRef, tyPtr}):
       result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
                  up, env.info))
+      # That assignment stores a real `ref`, so `injectDestructorCalls` has to
+      # find the up-field type's ops — otherwise it stays a raw pointer store,
+      # the enclosing env's refcount is one too low, and at teardown the two
+      # envs' mutually recursive `=destroy`s each believe they hold the last
+      # reference and recurse until the stack is gone. Whole-program cgen never
+      # noticed: some LATER lifting pass creates this very ref type's ops, and it
+      # runs before any routine's destructor injection. The per-module backend
+      # injects a routine right after lifting it (the `lower` stage), long before
+      # the module's top level is transformed at all (that is `cg`).
+      if up.typ != nil and up.typ.kind == tyRef and up.typ.elementType != nil:
+        createTypeBoundOpsLL(d.graph, up.typ, env.info, d.idgen, owner)
     #elif oldenv != nil and oldenv.typ == upField.typ:
     #  result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
     #             oldenv, env.info))
@@ -732,6 +743,10 @@ proc closureCreationForIter(owner: PSym, iter: PNode;
     if u != nil and u.typ.skipTypes({tyOwned, tyRef, tyPtr}) == expectedUpTyp:
       result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
                  u, iter.info))
+      # See the identical call in `rawClosureCreation`: the up-field's ops must
+      # exist by the time this assignment is destructor-injected.
+      if u.typ != nil and u.typ.kind == tyRef and u.typ.elementType != nil:
+        createTypeBoundOpsLL(d.graph, u.typ, iter.info, d.idgen, owner)
     else:
       localError(d.graph.config, iter.info, "internal error: cannot create up reference for iter")
   result.add makeClosure(d.graph, d.idgen, iter.sym, vnode, iter.info)

compiler/modulegraphs.nim

@@ -574,12 +574,6 @@ proc logGenericInstance*(g: ModuleGraph; inst: PSym) =
     let ownerModule = inst.itemId.module.int
     g.opsLog.add LogEntry(kind: GenericInstEntry, module: ownerModule, sym: inst)
 
-const
-  InstanceDisambBit* = 0x4000_0000'i32
-    ## Set in the `disamb` of routine instances whose value is content-derived
-    ## (see `setInstanceDisamb`); keeps them disjoint from the small counter
-    ## range ordinary symbols draw from, so the NIF name `name.disamb.module`
-    ## stays collision-free within a module.
 
 proc setInstanceDisamb*(g: ModuleGraph; inst, generic: PSym;
                         concreteTypes: openArray[PType]) =
@@ -618,12 +612,6 @@ proc setInstanceDisamb*(g: ModuleGraph; inst, generic: PSym;
       break
   inst.disamb = h
 
-const
-  HookDisambBit* = 0x2000_0000'i32
-    ## Set in the `disamb` of synthesized type-bound operators and `$enum`
-    ## procs whose value is content-derived (see `setHookDisamb`); disjoint
-    ## from both the small counter range and the `InstanceDisambBit` range.
-
 proc setHookDisamb*(g: ModuleGraph; hook: PSym; opName: string; typ: PType) =
   ## Under IC, replace a synthesized hook's counter-based `disamb` with a
   ## content-derived one: a hash of the operation name plus the `typeKey` of

compiler/options.nim

@@ -29,7 +29,7 @@ const
 
   nimEnableCovariance* = defined(nimEnableCovariance)
 
-  icFormatVersion* = "37"
+  icFormatVersion* = "38"
     ## Version of the IC cache format (the sem-NIF module layout written by
     ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
     ## that layout changes: `commandIc` wipes a nimcache whose `ic.version`

tests/ic/tclosure_hooks.nim

@@ -0,0 +1,101 @@
+discard """
+  description: '''IC vs `nim c`: closure environments, their hooks and their owners'''
+"""
+
+#? metamorphic
+
+# A closure's environment type — and the `=destroy`/`=copy` the compiler lifts
+# for it — is minted by the BACKEND, during the `lower` stage, and exists in no
+# module's semmed NIF. The per-module backend has to decide which translation
+# unit emits such a routine, and the owner walk it uses lands on the module of
+# the ORIGINAL generic: for a generic closure iterator defined in one module and
+# instantiated in another, that is a module which never sees the instance, so the
+# env's `=destroy` was emitted by nobody (`undefined reference to
+# eqdestroy__c485__…`). Every referencing TU emits it now.
+#
+# The steps then move the captured state around, because the env's LAYOUT is what
+# decides whether those hooks are trivial: a body-only edit that adds a capture
+# changes the env type of a routine whose importers do not re-sem.
+
+#!FILE clleaf.nim
+type Ev* = proc (s: string): string {.closure.}
+
+proc leafMaker*(tag: string): Ev =
+  var n = 0
+  proc outer(s: string): string =
+    proc inner(t: string): string =
+      inc n
+      tag & ":" & t & ":" & $n
+    inner(s)
+  result = outer
+
+iterator leafIter*[T](xs: seq[T]): T {.closure.} =
+  for x in xs: yield x
+
+#!FILE clmid.nim
+import clleaf
+
+proc midMaker*(tag: string): Ev =
+  let base = leafMaker(tag & "/mid")
+  var calls = 0
+  result = proc (s: string): string =
+    inc calls
+    base(s) & "#" & $calls
+
+proc midIter*(): seq[string] =
+  # instantiates `leafIter[string]` HERE, not where it is defined
+  result = @[]
+  for x in leafIter(@["p", "q"]): result.add x
+
+#!FILE main.nim
+import clleaf, clmid
+
+let t = midMaker("top")
+echo t("Alpha")
+echo t("Beta")
+echo midIter()
+
+# an instance only the main module has
+var fs: seq[float] = @[]
+for x in leafIter(@[1.5, 2.5]): fs.add x
+echo fs
+#!STEP
+
+# body-only edit that GROWS the environment: a second captured local
+#!FILE clleaf.nim
+type Ev* = proc (s: string): string {.closure.}
+
+proc leafMaker*(tag: string): Ev =
+  var n = 0
+  var seen: seq[string] = @[]
+  proc outer(s: string): string =
+    proc inner(t: string): string =
+      inc n
+      seen.add t
+      tag & ":" & t & ":" & $n & ":" & $seen.len
+    inner(s)
+  result = outer
+
+iterator leafIter*[T](xs: seq[T]): T {.closure.} =
+  var i = 0
+  for x in xs:
+    inc i
+    yield x
+#!STEP
+
+# and shrink it again
+#!FILE clleaf.nim
+type Ev* = proc (s: string): string {.closure.}
+
+proc leafMaker*(tag: string): Ev =
+  var n = 0
+  proc outer(s: string): string =
+    proc inner(t: string): string =
+      inc n
+      tag & ":" & t & ":" & $n
+    inner(s)
+  result = outer
+
+iterator leafIter*[T](xs: seq[T]): T {.closure.} =
+  for x in xs: yield x
+#!STEP

tests/ic/tclosure_nested_iter.nim

@@ -0,0 +1,90 @@
+discard """
+  description: '''IC vs `nim c`: a closure iterator nested in a closure iterator'''
+"""
+
+#? metamorphic
+
+# `env.:up = enclosingEnv` links a nested routine's environment to its parent,
+# and the two environments then reference each other. That assignment has to go
+# through `=copy` (with the cyclic increment) or the parent's refcount is one too
+# low, and at teardown both `=destroy`s believe they hold the last reference and
+# recurse until the stack is gone — a SIGSEGV, after the program's own output has
+# already been printed. (`tests/iter/tnestedclosures.nim`, "Test 3".)
+#
+# Whether it becomes a `=copy` depends on the up-field type's hooks existing when
+# the routine is destructor-injected. Whole-program cgen got that for free: a
+# LATER lifting pass creates them, and it runs before any routine's injection.
+# The per-module backend injects a routine right after lifting it (the `lower`
+# stage), long before the module's top level is transformed at all (that is
+# `cg`) — so the hooks are created at the assignment site now.
+
+#!FILE main.nim
+iterator foo(): int {.closure.} =
+  let x = 34
+  proc bar() = echo "bar sees ", x
+  iterator bar2(): int {.closure.} =
+    bar()
+    yield x
+  for y in bar2():
+    yield y
+
+for v in foo(): echo v
+
+# a closure iterator nested in a closure iterator, inside a proc
+proc factory() =
+  iterator outerIt(): int {.closure.} =
+    iterator innerIt(): int {.closure.} =
+      yield 0
+      yield 1
+      yield 2
+    for x in innerIt(): yield x
+  for x in outerIt(): echo x
+factory()
+
+# the iterator's env outlives the proc that made it
+proc keep(): iterator (): string =
+  let held = "kept"
+  result = iterator (): string =
+    yield held
+    yield held & "!"
+for s in keep()(): echo s
+#!STEP
+
+# growing the captured state changes both env layouts
+#!FILE main.nim
+iterator foo(): int {.closure.} =
+  let x = 34
+  var log: seq[string] = @[]
+  proc bar() =
+    log.add "bar"
+    echo "bar sees ", x, " ", log.len
+  iterator bar2(): int {.closure.} =
+    bar()
+    bar()
+    yield x
+  for y in bar2():
+    yield y
+
+for v in foo(): echo v
+
+proc factory() =
+  iterator outerIt(): int {.closure.} =
+    var emitted = 0
+    iterator innerIt(): int {.closure.} =
+      yield 0
+      yield 1
+      yield 2
+    for x in innerIt():
+      inc emitted
+      yield x * emitted
+  for x in outerIt(): echo x
+factory()
+
+proc keep(): iterator (): string =
+  let held = "kept"
+  let extra = "+"
+  result = iterator (): string =
+    yield held & extra
+    yield held & "!" & extra
+for s in keep()(): echo s
+#!STEP