preparing the IC core for binary bif files

compiler/ccgtypes.nim

@@ -1807,6 +1807,16 @@ proc generateRttiDestructor(g: ModuleGraph; typ: PType; owner: PSym; kind: TType
 
   incl result.flagsImpl, sfFromGeneric
   incl result.flagsImpl, sfGeneratedOp
+  # Under IC the `rttiDestroy` wrapper is generated independently in every cg
+  # process that emits `typ`'s RTTI (the type-info is emit-everywhere). A plain
+  # counter `disamb` renumbers per process, so the RTTI table baked in module A
+  # references `rttiDestroy_c<n>` while module B (the =destroy owner) defines a
+  # different number → undefined at link. Give it a content-derived `disamb`
+  # (stable across processes) + `HookDisambBit`, exactly like `symPrototype` does
+  # for the hook itself: same `typ` ⇒ same C name everywhere, and the bit makes
+  # `emitsBodyInThisModule` emit the body in every demander (merge dedups). The
+  # `"rttiDestroy"` op-name keeps its key disjoint from the real `=destroy` hook's.
+  setHookDisamb(g, result, "rttiDestroy", typ)
 
 proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp; result: var Builder) =
   let theProc = getAttachedOp(m.g.graph, t, op)

compiler/ccgutils.nim

@@ -11,7 +11,7 @@
 
 import
   ast, types, msgs, wordrecg,
-  platform, trees, options, cgendata, mangleutils, renderer
+  platform, trees, options, cgendata, mangleutils, renderer, modulegraphs
 
 import std/[hashes, strutils, formatfloat]
 
@@ -116,7 +116,17 @@ proc makeUnique(m: BModule; s: PSym, name: string = ""): string =
   # restarts at 0 and would collide with loaded symbols' ids
   if s.itemId.isBackendMinted:
     result.add "_c"
-    result.add $s.itemId.item
+    if (s.disamb and HookDisambBit) != 0'i32:
+      # A backend-minted sym whose `disamb` is content-derived (setHookDisamb gave
+      # it HookDisambBit) — e.g. the `rttiDestroy` wrapper. Its `itemId.item` is a
+      # PER-PROCESS backend counter, so using it makes the C name diverge across
+      # the emit-everywhere processes: the type's RTTI table (emit-everywhere,
+      # merge-deduped) ends up referencing one process's `_c<item>` while the
+      # wrapper is defined with another's -> undefined at link (`rttiDestroy_c23`).
+      # The content-derived disamb is stable across processes, so use it.
+      result.add $s.disamb
+    else:
+      result.add $s.itemId.item
   else:
     result.add "_u"
     # Mirror `mangleProcNameExt`: use the per-(module,name) `disamb`, NOT

compiler/cgen.nim

@@ -1218,8 +1218,17 @@ proc closeNamespaceNim(result: var Builder) =
 
 proc closureSetup(p: BProc, prc: PSym) =
   if tfCapturesEnv notin prc.typ.flags: return
-  # prc.ast[paramsPos].last contains the type we're after:
-  var ls = lastSon(prc.ast[paramsPos])
+  # prc.ast[paramsPos].last contains the type we're after — BUT a closure loaded
+  # from a `.t.bif` (a lambda-lifted nested proc / generic instance the `lower`
+  # stage transformed) can arrive with an EMPTY AST param node: the lifted hidden
+  # `:env` param lives in `typ.n`, the authoritative signature (`genProc` already
+  # reads `typ.n`, not the AST). The two param nodes diverge across the NIF
+  # boundary; fall back to `typ.n` so the env param resolves instead of indexing
+  # an empty container.
+  var params = prc.ast[paramsPos]
+  if params.safeLen == 0 and prc.typ.n != nil and prc.typ.n.kind == nkFormalParams:
+    params = prc.typ.n
+  var ls = lastSon(params)
   if ls.kind != nkSym:
     internalError(p.config, prc.info, "closure generation failed")
   var env = ls.sym
@@ -1464,8 +1473,16 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
   var returnStmt: Snippet = ""
   assert(prc.ast != nil)
 
+  # A body LOADED from `.t.bif` was already FULLY lowered by the `lower` stage —
+  # transformed AND destructor-injected (see nifbackend.generateLowerStage). The
+  # `.t.bif` is the authoritative backend artifact; re-injecting here would lower
+  # it twice (double `=destroy` calls) and, worse, re-lift the env hooks per cg
+  # process (owned by nobody → undefined at link). So inject ONLY when the body
+  # was re-derived in this process (`wasLoaded == false`). Capture before
+  # `transformBody`, which returns the cached body (non-nil) when it was loaded.
+  let wasLoaded = prc.transformedBody != nil
   var procBody = transformBody(m.g.graph, m.idgen, prc, {})
-  if sfInjectDestructors in prc.flags:
+  if sfInjectDestructors in prc.flags and not wasLoaded:
     procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody)
 
   let tmpInfo = prc.info

compiler/deps.nim

@@ -16,6 +16,7 @@ import options, msgs, lineinfos, pathutils, condsyms,
 
 import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder]
 import "../dist/nimony/src/gear2" / modnames
+import icnifcore
 
 type
   FilePair = object
@@ -51,24 +52,24 @@ proc parsedFile(c: DepContext; f: FilePair): string =
   getNimcacheDir(c.config).string / f.modname & ".p.nif"
 
 proc semmedFile(c: DepContext; f: FilePair): string =
-  getNimcacheDir(c.config).string / f.modname & ".s.nif"
+  getNimcacheDir(c.config).string / f.modname & ".s.bif"
 
 proc ifaceFile(c: DepContext; f: FilePair): string =
   ## Interface-cookie sidecar written by `nim m` (ast2nif.writeIfaceCookie,
   ## OnlyIfChanged). Dependents' nim_m rules use it as their input instead of
   ## the semmed NIF: a body-only change in a dependency then keeps the sidecar
   ## mtime and nifmake prunes the whole re-sem cascade behind it.
-  getNimcacheDir(c.config).string / f.modname & ".iface.nif"
+  getNimcacheDir(c.config).string / f.modname & ".iface.bif"
 
 proc implFile(c: DepContext; suffix: string): string =
   ## Implementation-cookie sidecar (ast2nif.writeImplCookie): flips on ANY
   ## content change of the module (private bodies included; supersedes the
   ## iface cookie). Used as the edge for dependents that consumed the
   ## module's bodies at compile time (NeedsImpl edges).
-  getNimcacheDir(c.config).string / suffix & ".impl.nif"
+  getNimcacheDir(c.config).string / suffix & ".impl.bif"
 
 proc edgesFile(c: DepContext; f: FilePair): string =
-  getNimcacheDir(c.config).string / f.modname & ".edges.nif"
+  getNimcacheDir(c.config).string / f.modname & ".edges.bif"
 
 proc readNeedsImpl(c: DepContext; f: FilePair): seq[string] =
   ## Reads the module's recorded NeedsImpl edge set (module suffixes whose
@@ -79,19 +80,10 @@ proc readNeedsImpl(c: DepContext; f: FilePair): seq[string] =
   ## gated input of its rule, so the rule re-fires and re-records.
   result = @[]
   if fileExists(c.edgesFile(f)):
-    var s = nifstreams.open(c.edgesFile(f))
-    try:
-      discard processDirectives(s.r)
-      while true:
-        let t = next(s)
-        if t.kind == EofToken: break
-        if t.kind == StringLit:
-          result.add pool.strings[t.litId]
-    finally:
-      close s
+    result = collectBifStrLits(c.edgesFile(f))
 
 proc semDepsFile(c: DepContext; f: FilePair): string =
-  getNimcacheDir(c.config).string / f.modname & ".s.deps.nif"
+  getNimcacheDir(c.config).string / f.modname & ".s.deps.bif"
 
 proc readSemDeps(c: DepContext; f: FilePair): seq[string] =
   ## The module's REAL direct imports (full source paths) as sem resolved them,
@@ -99,16 +91,7 @@ proc readSemDeps(c: DepContext; f: FilePair): seq[string] =
   ## (ast2nif.writeSemDeps). Missing file (not yet semmed) -> empty.
   result = @[]
   if fileExists(c.semDepsFile(f)):
-    var s = nifstreams.open(c.semDepsFile(f))
-    try:
-      discard processDirectives(s.r)
-      while true:
-        let t = next(s)
-        if t.kind == EofToken: break
-        if t.kind == StringLit:
-          result.add pool.strings[t.litId]
-    finally:
-      close s
+    result = collectBifStrLits(c.semDepsFile(f))
 
 proc findNifler(): string =
   # Look for nifler in common locations
@@ -981,12 +964,12 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
   var cFiles = newSeq[string](c.nodes.len)
   var tFiles = newSeq[string](c.nodes.len)
   # The `lower` stage writes a PROPER module NIF the cg/emit stages load via
-  # `toNifFilename` (a `.s.nif` sibling), so its `.t.nif` lives at the suffix base
+  # `toNifFilename` (a `.s.bif` sibling), so its `.t.bif` lives at the suffix base
   # (mirroring `semmedFile`), not next to the throwaway `.c`.
   for i, node in c.nodes:
     cFiles[i] = backendCFile(c, node)
     cnifFiles[i] = cFiles[i] & ".nif"
-    tFiles[i] = nimcache / node.files[0].modname & ".t.nif"
+    tFiles[i] = nimcache / node.files[0].modname & ".t.bif"
 
   # Only code-generate modules the real program actually reaches; statically
   # over-approximated nodes (e.g. `winlean` on Linux) are sem'd but not emitted.

compiler/lambdalifting.nim

@@ -309,6 +309,19 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
       [s.name.s, owner.name.s, $owner.typ.callConv])
   unsealForTransform(owner.typ)
   incl(owner.typ, tfCapturesEnv)
+  # A closure proc type that captures an env owns a REF to it: copying the closure
+  # value must incref the env and destroying it must decref. That is exactly what
+  # `tfHasAsgn` signals to `injectDestructorCalls` (so a closure assignment becomes
+  # `=copy`, not a raw field store). Set it HERE, at closure-type creation, so the
+  # flag is DETERMINISTIC and serializes with the type (writeTypeDef) — rather than
+  # depending on it being set as a side effect of the first `createTypeBoundOps`
+  # lift (liftdestructors ~1498, "XXX Breaks IC!"). Under IC the per-module `lower`
+  # stage is a separate process that lowers routines in index order; if a consumer
+  # (e.g. `workNimAsyncContinue`) was lowered before the closure type's ops were
+  # lifted, the env store emitted a RAW assign with no incref → the env was freed
+  # before the async callback ran → "yielded `nil`". Setting it at creation fixes
+  # that for both the in-process and the loaded (`.t.bif`) consumer.
+  incl(owner.typ, tfHasAsgn)
   if not isEnv:
     owner.typ.callConv = ccClosure
 

compiler/mangleutils.nim

@@ -61,12 +61,22 @@ proc mangleProcNameExt*(graph: ModuleGraph, s: PSym): string =
     # starts with an EMPTY per-name disamb table, so its `disamb` restarts at 0
     # and collides with same-named sem-time symbols loaded from NIFs (two
     # `=destroy` hooks both mangling to `_u2` → "conflicting types for ..." in
-    # the generated C). These symbols never cross a process boundary (nifc
+    # the generated C). Most such symbols never cross a process boundary (nifc
     # lifts, emits and compiles them in one run), so the per-module-unique
     # item id is a safe and deterministic discriminator; the `_c` marker keeps
     # the namespace disjoint from `_u<disamb>`.
     result = "_c"
-    result.addInt s.itemId.item
+    if (s.disamb and HookDisambBit) != 0'i32:
+      # EXCEPTION: a backend-minted sym whose `disamb` is content-derived
+      # (setHookDisamb gave it HookDisambBit) — e.g. the `rttiDestroy` wrapper —
+      # DOES cross process boundaries: its C name is baked into the type's RTTI
+      # table, which is emit-everywhere and merge-deduped, so one process's
+      # `_c<item>` (a per-process backend counter) ends up referenced while the
+      # wrapper is defined with another's → undefined at link (`rttiDestroy_c23`).
+      # The content-derived disamb is stable across processes; use it.
+      result.addInt s.disamb
+    else:
+      result.addInt s.itemId.item
   else:
     result = "_u"
     # Use `disamb` rather than `itemId.item`: under incremental compilation a

compiler/nifbackend.nim

@@ -27,6 +27,7 @@ import ast, options, lineinfos, modulegraphs, cgendata, cgen,
   cnif
 from cgmeth import generateIfMethodDispatchers
 from transf import transformBody
+from injectdestructors import injectDestructorCalls
 import ic / replayer
 
 proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex;
@@ -156,7 +157,7 @@ proc signatureHasMetaType(t: PType; depth: int = 0): bool =
   for k in t.kids:
     if signatureHasMetaType(k, depth + 1): return true
 
-proc ownsRuntimeRoutine(s: PSym; modPos: int): bool =
+proc ownsRuntimeRoutine(s: PSym; modPos: int; forLowering = false): bool =
   ## A concrete, non-generic, runtime routine with a real body, OWNED by the
   ## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a
   ## routine called only from other modules is still emitted by somebody) and
@@ -178,12 +179,21 @@ proc ownsRuntimeRoutine(s: PSym; modPos: int): bool =
   ## A `{.closure.}` iterator IS a standalone runtime routine (unlike an inline
   ## iterator, which is expanded at each call site) and must be emitted by its
   ## owner — else a cross-module `for` over it links to nothing.
+  ##
+  ## `forLowering`: the `lower` stage must ALSO transform the generic INSTANCES
+  ## this module serializes into its `.t.bif` (each demander keeps its own copy,
+  ## `itemId.module == modPos`). The `.t.bif` is the authoritative backend
+  ## artifact — every routine `cg` emits must arrive with its lowered body baked
+  ## in, NEVER re-derived in `cg` (re-derivation on the partially-loaded backend
+  ## state is exactly what crashed `newSelector`). The `cg`/emit-everywhere path
+  ## keeps the `sfFromGeneric` exclusion (instances are still deduped by content
+  ## name at merge); only the body-producing `lower` pass relaxes it.
   s.itemId.module == modPos and
   (s.kind in {skProc, skFunc, skConverter, skMethod} or
    (s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and
   s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
   s.magic == mNone and
-  sfFromGeneric notin s.flags and
+  (forLowering or sfFromGeneric notin s.flags) and
   sfDispatcher notin s.flags and
   {sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
   s.typ != nil and not signatureHasMetaType(s.typ) and
@@ -367,12 +377,34 @@ proc setNestedClosureBodies(g: ModuleGraph; idgen: IdGenerator; n: PNode;
   if n.kind == nkSym:
     let s = n.sym
     if s != nil and s.kind in routineKinds and s != owner and
+        s.skipGenericOwner != nil and s.skipGenericOwner.kind != skModule and
         not seen.containsOrIncl(s.id):
-      if s.ast != nil and getBody(g, s).kind != nkEmpty and
-          s.typ != nil and s.typ.callConv == ccClosure:
-        if s.transformedBody == nil:
+      # Covers ALL nested routines, not only ccClosure ones. A NIMCALL nested proc
+      # the async transform mints (e.g. workNimAsyncContinue) already has its
+      # lifted body set by the OWNER's transformBody, but it is NOT in the owned
+      # loop (owner is a proc, not the module). Without injecting it HERE it is
+      # serialized transform-only; cg loads it (wasLoaded) and skips injection, so
+      # a closure-env store stays a raw field assign with no incref -> the env is
+      # freed before the async callback runs -> "yielded nil". `seen` (shared
+      # across the owned loop) injects each routine exactly once.
+      if s.ast != nil and getBody(g, s).kind != nkEmpty:
+        # Only ccClosure routines are safe to `transformBody` standalone here; a
+        # nimcall nested proc already has its lifted body from the owner's lift,
+        # and transforming an arbitrary nested routine with no cached body crashes
+        # (not in a standalone-transformable state).
+        let weTransformed = s.transformedBody == nil and
+                            s.typ != nil and s.typ.callConv == ccClosure
+        if weTransformed:
           s.transformedBody = transformBody(g, idgen, s, {})
-        setNestedClosureBodies(g, idgen, s.transformedBody, s, seen)
+        if s.transformedBody != nil:
+          # Inject destructors so cg loads a fully-lowered body and never rebuilds
+          # (mirrors non-IC, which injects every nested proc separately). The
+          # importer `n2` skField collision this used to trigger is fixed at the
+          # NIF-naming layer (toNifSymName gives derived env fields a unique
+          # disamb), so injecting ccClosure nested procs here is safe.
+          if sfInjectDestructors in s.flags:
+            s.transformedBody = injectDestructorCalls(g, idgen, s, s.transformedBody)
+          setNestedClosureBodies(g, idgen, s.transformedBody, s, seen)
   else:
     for i in 0 ..< n.safeLen:
       setNestedClosureBodies(g, idgen, n[i], owner, seen)
@@ -448,12 +480,19 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
   # (`=destroy` etc.) into `g.opsLog`; snapshot its length so we serialize exactly
   # the ops THIS stage created (not those loaded from `.s.nif`).
   let opsLogStart = g.opsLog.len
+  # Shared across the owned loop so a nested routine reachable from more than one
+  # owner is transformed + destructor-injected EXACTLY once (double injection
+  # would emit two `=destroy`/`=copy` runs).
+  var seenNested = initIntSet()
   for s in moduleSymbolStubs(ast.program, FileIndex modPos):
-    if ownsRuntimeRoutine(s, modPos):
-      # `.s.nif` wins: a routine already transformed during sem (CT eval / macro /
-      # VM transform) carries its lowered body in the `.s.nif` slot — don't
-      # re-transform it here.
-      if s.transformedBody != nil: continue
+    if ownsRuntimeRoutine(s, modPos, forLowering = true):
+      # REUSE path (`icReuseSemLowering` ON): a routine already transformed during
+      # sem (CT eval / macro / VM transform) carries its lowered body in the
+      # `.s.nif` slot (loaded into `transformedBody`) — don't re-transform it.
+      # Default OFF: the slot is never loaded (see loadSymFromCursor), so
+      # `transformedBody` is nil here and we always re-derive below. See
+      # doc/ic_backend_simplify.md §6a/§6b.
+      if icReuseSemLowering(g.config) and s.transformedBody != nil: continue
       # A routine serialized as a forward-decl + impl pair (writeSymDef's
       # "separate forward declaration and implementation") loads as TWO syms; the
       # impl `s` we transform here can carry body entities (`result`, locals,
@@ -469,9 +508,19 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
       # Retain the transformed body on the sym so `writeSymDef` serializes it in
       # the routine's `(sd)` 2-way-body slot.
       s.transformedBody = transformBody(g, tb.idgen, s, {})
-      # Cache the lifted body on nested ccClosure routines too, so a module-indexed
-      # nested closure serializes its lifted (capture-rewritten) body.
-      var seenNested = initIntSet()
+      # Run the destructor injection HERE so the `.t.bif` body is FULLY lowered:
+      # `injectDestructorCalls` is demand-driven (it decides where destructors go
+      # by move analysis) and LIFTS the type-bound ops it needs (e.g. a nested
+      # closure env's `=destroy`) into `g.opsLog` — which the `hooks` collection
+      # below then serializes. Done in `cg` instead, those ops were lifted per-cg
+      # process, owned by nobody, and emitted as a prototype-only → undefined at
+      # link (the `eqdestroy__c<n>` gap). cg must NOT re-inject a loaded body
+      # (see genProcLvl3's `wasLoaded` gate) so this stays the single injection.
+      if sfInjectDestructors in s.flags:
+        s.transformedBody = injectDestructorCalls(g, tb.idgen, s, s.transformedBody)
+      # Cache the lifted+injected body on nested ccClosure routines too, so a
+      # module-indexed nested closure serializes its lifted (capture-rewritten,
+      # destructor-injected) body.
       setNestedClosureBodies(g, tb.idgen, s.transformedBody, s, seenNested)
   # Collect the hooks this stage lifted, and transform each hook ROUTINE's body
   # too (it is itself lowered into NIFC). The hooks' `(sd)` + transformed body go
@@ -487,14 +536,18 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
       hooks.add e
       # Transform the hook routine's body and cache it on the sym so `writeSymDef`
       # serializes it in the hook's `(sd)` transformed-body slot (`transformBody
-      # {}` returns the body but does not cache it).
+      # {}` returns the body but does not cache it). Inject the hook's own
+      # destructors here too (it can destroy fields/temporaries) so cg loads a
+      # fully-lowered hook and never re-injects.
       e.sym.transformedBody = transformBody(g, tb.idgen, e.sym, {})
+      if sfInjectDestructors in e.sym.flags:
+        e.sym.transformedBody = injectDestructorCalls(g, tb.idgen, e.sym, e.sym.transformedBody)
     inc i
   # Re-serialize the whole module to its suffix-based `.t.nif` (the path
   # `toNifFilename` resolves for the cg/emit stages). `writeLoweredModule` seals
   # routines itself.
   let suffix = cachedModuleSuffix(g.config, FileIndex modPos)
-  let wholeArtifact = toGeneratedFile(g.config, AbsoluteFile(suffix), ".t.nif").string
+  let wholeArtifact = toGeneratedFile(g.config, AbsoluteFile(suffix), ".t.bif").string
   writeLoweredModule(ast.program, g.config, target, hooks, wholeArtifact)
   if isDefined(g.config, "icDceCheck"):
     stderr.writeLine "[icLower] " & extractFilename(wholeArtifact) & " " &

compiler/nim.cfg

@@ -14,6 +14,10 @@ define:nimPreviewAsmSemSymbol
 define:nimPreviewCStringComparisons
 #define:nimPreviewDuplicateModuleError
 # Incompatible with Nimony's compat2.nim for now
+# NOTE: `-d:virtualParRi` (jump-encoded ParLe + elided ParRi) is NOT yet enabled:
+# the IC writer assembles buffers by raw token splicing (`dest.add content[i]`),
+# which does not seal scopes the way `addParRi` does, so sealed `(stmts)` get
+# jump=0 and serialize empty. Enabling it needs writer buffer-sealing work first.
 
 threads:off
 

compiler/options.nim

@@ -29,7 +29,7 @@ const
 
   nimEnableCovariance* = defined(nimEnableCovariance)
 
-  icFormatVersion* = "21"
+  icFormatVersion* = "23"
     ## Version of the IC cache format (the sem-NIF module layout written by
     ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
     ## that layout changes: `commandIc` wipes a nimcache whose `ic.version`
@@ -793,6 +793,17 @@ template quitOrRaise*(conf: ConfigRef, msg = "") =
   else:
     quit(msg) # quits with QuitFailure
 
+proc icReuseSemLowering*(conf: ConfigRef): bool {.inline.} =
+  ## When ON, the per-module `lower` backend stage REUSES the VM/CT lowering that
+  ## sem cached in the `.s.nif` 2-way-body slot (the non-IC single-lowering
+  ## semantics) instead of re-deriving the transform. Default OFF: the backend
+  ## re-derives every body from the pristine semchecked body (simpler; allowed by
+  ## the 2026-06-27 spec that VM-requested frontend transforms need not influence
+  ## the backend). The switch exists so caching can be restored if a target (e.g.
+  ## Nimbus) depends on the cached lowering being reused, not re-derived. See
+  ## doc/ic_backend_simplify.md §6b.
+  isDefined(conf, "icReuseSemLowering")
+
 proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.ideActive or conf.cmd in cmdDocLike
 proc usesWriteBarrier*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc
 proc usesSso*(conf: ConfigRef): bool {.inline.} = conf.selectedStrings == stringSso