now async tests work with IC

compiler/ast2nif.nim

@@ -2236,6 +2236,20 @@ proc sealLoadedBackendEntities*(c: var DecodeContext) =
   for _, v in c.types:
     if v[0] != nil and v[0].state == Complete: v[0].state = Sealed
 
+proc sealLoadedRoutines*(c: var DecodeContext) =
+  ## Whole-module-lowering variant of `sealLoadedBackendEntities`: seal ONLY the
+  ## module's ROUTINE syms. A `.t.nif` written by `writeLoweredModule` is the
+  ## SOLE source the `cg` stage loads (there is no `.s.nif` fallback for its
+  ## bodies), so unlike the per-`(lowered)`-entry path, every type, global, param
+  ## and local must still emit a REAL def in it — only cross-routine references
+  ## may be `SymUse`s (each routine's def is emitted once, at module scope, by
+  ## the explicit stub loop). Types/globals stay `Complete` so `writeType`/
+  ## `writeGlobals` emit them; routines become `Sealed` so a body referencing
+  ## another routine writes a `SymUse` resolved through the module index.
+  for _, v in c.syms:
+    if v[0] != nil and v[0].state == Complete and v[0].kindImpl in routineKinds:
+      v[0].state = Sealed
+
 proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym
 
 proc repTagToOp(tagId: TagId): (bool, TTypeAttachedOp) =
@@ -2583,6 +2597,14 @@ proc moduleSymbolStubs*(c: var DecodeContext; module: FileIndex): seq[PSym] =
 
 proc toNifFilename*(conf: ConfigRef; f: FileIndex): string =
   let suffix = moduleSuffix(conf, f)
+  # The `cg`/`emit` backend stages load the lowered whole-module NIF (transformed
+  # bodies + lifted sigs baked in) when `-d:icWholeLowered` is on; the `lower`
+  # stage and the frontend (`cmdM`) still read the semchecked `.s.nif`.
+  if isDefined(conf, "icWholeLowered") and conf.cmd == cmdNifC and
+     (conf.icBackendStage == "cg" or conf.icBackendStage == "emit"):
+    let t = toGeneratedFile(conf, AbsoluteFile(suffix), ".t.nif").string
+    if fileExists(t):
+      return t
   result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.nif").string
 
 proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym =
@@ -2971,6 +2993,112 @@ proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: va
   let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f))
   result = loadNifModule(c, suffix, interf, interfHidden, flags)
 
+proc writeLoweredModule*(c: var DecodeContext; config: ConfigRef;
+                         precomp: PrecompiledModule;
+                         hooks: openArray[LogEntry]; outfile: string) =
+  ## Re-serialize a backend-loaded module as a FULL module NIF (`.t.nif`) whose
+  ## routine `(sd)` entries carry their TRANSFORMED bodies (the `lower` stage set
+  ## them, recursively lifting nested closures — including the async state-machine
+  ## procs whose inner closure the per-`(lowered)`-entry path failed to cross) and
+  ## whose lambda-lift-minted entities (closure-env types/syms, lifted nested
+  ## procs) are real, indexed defs. The `cg` stage then loads it through the
+  ## normal module loader (`moduleFromNifFile`), so a transformed body arrives via
+  ## `loadSymFromCursor`'s Step-A 2-way-body slot WITH the lifted signature — no
+  ## `(lowered)` side-car, no `:envP` re-weld. This realizes `ic_ideas.md`'s eager
+  ## two-way body whole-module.
+  let thisModule = precomp.module.positionImpl.int32
+  # Routines → Sealed (cross-routine refs become SymUse, defs emitted once below);
+  # types/globals/params/locals stay Complete and emit real defs (the `.t.nif` is
+  # the sole source the cg stage reads — no `.s.nif` fallback for them).
+  sealLoadedRoutines(c)
+  var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule)
+  w.inProc = 1
+  w.lowering = true
+  var content = createTokenBuf(300)
+  let rootInfo = trLineInfo(w, precomp.topLevel.info)
+  createStmtList(content, rootInfo)
+
+  # This module's ops (hooks/converters/methods/pure-enums) loaded from `.s.nif`,
+  # plus the type-bound ops the lower transform just lifted (closure-env
+  # `=destroy` etc., which have no `.s.nif` entry).
+  for op in precomp.logOps:
+    if op.module == thisModule.int:
+      writeOp(w, content, op)
+  for op in hooks:
+    writeOp(w, content, op)
+
+  var bottom = createTokenBuf(300)
+  # Imperative init code + global let/var/const sections + replay actions — all
+  # that a backend-loaded `topLevel` carries (routines are lazy index sdefs, not
+  # here). Emits + seals the module's globals.
+  w.writeToplevelNode content, bottom, precomp.topLevel
+
+  # Routine DEFS with transformed bodies, sourced from the index.
+  for s in moduleSymbolStubs(c, FileIndex thisModule):
+    if s.kindImpl in routineKinds and s.itemId.module == thisModule:
+      writeSymDef(w, bottom, s)
+
+  # Lifted hook ROUTINES (`@bk`, NEW in the lower stage — no `.s.nif` sdef, so
+  # absent from `moduleSymbolStubs`): emit each as a full def (sig + transformed
+  # body) so `injectDestructorCalls` in cg resolves the loaded env's `=destroy`.
+  var emittedHooks = initHashSet[int32]()
+  for op in hooks:
+    if op.sym != nil and op.sym.kindImpl in routineKinds and
+        not emittedHooks.containsOrIncl(op.sym.itemId.item):
+      writeSymDef(w, bottom, op.sym)
+
+  # deps / reexports / offers — mirror writeNifModule so the cg backend closure
+  # walk, interface re-export and generic-instance reuse all work off `.t.nif`.
+  for dep in precomp.deps:
+    if not w.depSuffixes.containsOrIncl(dep.string):
+      w.deps.addParLe importTag, NoLineInfo
+      w.deps.addDotToken
+      w.deps.addDotToken
+      w.deps.addStrLit dep.string
+      w.deps.addParRi
+  for (mname, msuffix) in precomp.reexportedModules:
+    w.deps.addParLe reexpModTag, NoLineInfo
+    w.deps.addStrLit mname
+    w.deps.addStrLit msuffix
+    w.deps.addParRi
+  for off in precomp.genericOffers:
+    w.deps.addParLe offerTag, NoLineInfo
+    w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.generic)), NoLineInfo
+    w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.inst)), NoLineInfo
+    w.deps.addIntLit off.genericParamsCount
+    for ct in off.concreteTypes:
+      w.deps.addSymUse pool.syms.getOrIncl(typeToNifSym(ct, w.infos.config)), NoLineInfo
+    w.deps.addParRi
+  for off in precomp.typeOffers:
+    w.deps.addParLe typeOfferTag, NoLineInfo
+    w.deps.addStrLit w.toNifSymName(off.generic)
+    w.deps.addStrLit typeToNifSym(off.inst, w.infos.config)
+    w.deps.addParRi
+  # OWNER MUST EMIT offered types this module owns (see writeNifModule).
+  for off in precomp.genericOffers:
+    for ct in off.concreteTypes:
+      if ct != nil and ct.uniqueId.module == w.currentModule and ct.state == Complete:
+        writeType(w, bottom, ct)
+  for off in precomp.typeOffers:
+    if off.inst != nil and off.inst.uniqueId.module == w.currentModule and
+        off.inst.state == Complete:
+      writeType(w, bottom, off.inst)
+
+  # Assemble exactly as writeNifModule: (stmts . . <deps> <ops+toplevel>
+  # (implementation) <bottom> ).
+  content.addParLe implTag, NoLineInfo
+  content.addParRi()
+  content.add bottom
+  content.addParRi()
+
+  var dest = createTokenBuf(600)
+  createStmtList(dest, rootInfo)
+  dest.add w.deps
+  for i in 3 ..< content.len-1:
+    dest.add content[i]
+  dest.addParRi()
+  writeFile(dest, outfile)
+
 when isMainModule:
   import std / syncio
   let obj = parseSymName("a.123.sys")

compiler/deps.nim

@@ -942,10 +942,17 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
   var cnifFiles = newSeq[string](c.nodes.len)
   var cFiles = newSeq[string](c.nodes.len)
   var tFiles = newSeq[string](c.nodes.len)
+  # Whole-module lowering writes a PROPER module NIF the cg/emit stages load via
+  # `toNifFilename` (a `.s.nif` sibling), so its `.t.nif` lives at the suffix base
+  # (mirroring `semmedFile`), not next to the throwaway `.c`. The side-car path
+  # keeps the `.c`-relative name (loaded directly by `registerLoweredModule`).
+  let wholeMode = isDefined(c.config, "icWholeLowered")
   for i, node in c.nodes:
     cFiles[i] = backendCFile(c, node)
     cnifFiles[i] = cFiles[i] & ".nif"
-    tFiles[i] = cFiles[i] & ".t.nif"
+    tFiles[i] =
+      if wholeMode: nimcache / node.files[0].modname & ".t.nif"
+      else: cFiles[i] & ".t.nif"
 
   var b = nifbuilder.open(result)
   defer: b.close()
@@ -1012,7 +1019,13 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
     inputStr mainNif
     for n2 in c.nodes:
       inputStr c.semmedFile(n2.files[0])
-    inputStr tFiles[i]
+    # Whole-module mode loads dependencies FROM their `.t.nif` (toNifFilename), so
+    # every module's lowered NIF must precede this cg rule; the side-car only needs
+    # this module's own `.t.nif`.
+    if wholeMode:
+      for j in 0 ..< c.nodes.len: inputStr tFiles[j]
+    else:
+      inputStr tFiles[i]
     if node.id == 0:
       for j in 0 ..< c.nodes.len:
         if c.nodes[j].id != 0:
@@ -1040,6 +1053,10 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
     inputStr mainNif
     inputStr cnifFiles[i]
     inputStr mergeFile
+    # emit loads modules like cg (for getCFile/type resolution); under whole mode
+    # those come from the `.t.nif`s, so they must precede this rule.
+    if wholeMode:
+      for j in 0 ..< c.nodes.len: inputStr tFiles[j]
     outputStr cFiles[i]
     b.endTree()
 

compiler/nifbackend.nim

@@ -405,6 +405,10 @@ proc applyLoweredBodies(g: ModuleGraph; modules: seq[PrecompiledModule];
   ## Register every loaded module's `.t.nif` (env entities + lifted hooks),
   ## applying transformed bodies only for the cg target.
   if not icLoweredBodies(g.config): return  # Stage 0 (lazy): nothing to apply
+  # Whole-module mode: the cg stage already loaded each module FROM its `.t.nif`
+  # (toNifFilename), so transformed bodies arrive via loadSymFromCursor and lifted
+  # hooks via moduleFromNifFile's registerLoadedHooks — no side-car to apply.
+  if isDefined(g.config, "icWholeLowered"): return
   if precompSys.module != nil:
     registerLoweredModule(g, precompSys.module, applyBodies = false)
   for m in modules:
@@ -440,12 +444,18 @@ proc collectNestedClosureBodies(g: ModuleGraph; idgen: IdGenerator; n: PNode;
       collectNestedClosureBodies(g, idgen, n[i], owner, seen, entries)
 
 proc reownFromTwin(n: PNode; twin, s: PSym) =
-  ## Re-own to `s` every entity the frontend wrongly attributed to `s`'s
-  ## forward-decl `twin` (see the lower-stage loop). `twin` is one specific sym,
-  ## so only the mis-owned entities of THIS routine match — nested routines and
-  ## their own locals (owned by the nested routine, not `twin`) are untouched.
+  ## Re-own to `s` every entity the frontend attributed to `s`'s forward-decl
+  ## `twin` (found via the result's owner). lambda-lifting compares owners by
+  ## reference, so a twin-owned `result` is rejected as `illegalCapture`
+  ## ("'result' ... cannot be captured") and, once that is fixed, twin-owned
+  ## locals go missing from `s`'s env ("environment misses: ..."). Both are
+  ## pervasive on chronos `{.async.}` methods. Re-owning to `s` matches the
+  ## single-sym non-IC case. `twin` is ONE specific sym, so only THIS routine's
+  ## result-twin-owned entities match — re-owning entities of OTHER same-name
+  ## twins proved too blunt (it disrupts env construction and reintroduces the
+  ## very capture errors it should fix). `n.sym != s` guards self-ownership.
   if n == nil: return
-  if n.kind == nkSym and n.sym != nil and n.sym.owner == twin:
+  if n.kind == nkSym and n.sym != nil and n.sym != s and n.sym.owner == twin:
     setOwner(n.sym, s)
   for i in 0 ..< n.safeLen:
     reownFromTwin(n[i], twin, s)
@@ -498,6 +508,14 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
     # cg call (cgen.nim:1409); we keep only its return value (it clears
     # `transformedBody` for non-cached procs). `injectDestructorCalls` is NOT run
     # — it stays in `cg` on the loaded body.
+    # Whole-module mode (`-d:icWholeLowered`): re-serialize the ENTIRE module as a
+    # proper indexed NIF (`writeLoweredModule`) with transformed bodies baked into
+    # the routine sdefs, instead of the per-routine `(lowered)` side-car. cg loads
+    # it through the normal module loader, so nested procs (incl. async state
+    # machines) arrive as real defs with their lifted bodies — no re-weld, no
+    # `(lowered)` entries. `entries` stays unused; we only need `transformedBody`
+    # set on each routine sym so `writeSymDef` serializes it.
+    let wholeMode = isDefined(g.config, "icWholeLowered")
     var entries: seq[tuple[name: string; body: PNode]] = @[]
     # `transformBody`/lambda-lifting LIFTS the closure env's type-bound ops
     # (`=destroy` etc.) into `g.opsLog`; snapshot its length so we can serialize
@@ -513,20 +531,24 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
         # "separate forward declaration and implementation" design) loads as TWO
         # syms; the impl `s` we transform here can carry body entities (`result`,
         # locals, nested routines) owned by its fwd-decl TWIN, not by `s`.
-        # lambda-lifting compares owners by reference: `detectCapturedVars`
-        # rejects a twin-owned `result` as `illegalCapture` ("'result' ... cannot
-        # be captured") and, once that is fixed, the lifting pass can't find a
-        # twin-owned captured local in `s`'s env ("environment misses: ..."). Both
-        # are pervasive on chronos `{.async.}` methods. Re-own every twin-attributed
-        # entity to `s`, matching the single-sym non-IC case. The twin is a
-        # specific sym (found via the result's owner), so only THIS routine's
-        # mis-owned entities match. Backend-only (the lowered body is a `.t.nif`
-        # artifact), so frontend effect/exception inference is untouched.
+        # lambda-lifting compares owners by reference → `illegalCapture` rejects a
+        # twin-owned `result` and the lifting pass can't find twin-owned locals in
+        # `s`'s env. Pervasive on chronos `{.async.}` methods. Re-own them to `s`,
+        # matching the single-sym non-IC case. Backend-only (the lowered body is a
+        # `.t.nif` artifact), so frontend effect/exception inference is untouched.
         if s.ast != nil and s.ast.len > resultPos and
             s.ast[resultPos].kind == nkSym and s.ast[resultPos].sym.owner != s:
           reownFromTwin(s.ast, s.ast[resultPos].sym.owner, s)
         let tbody = transformBody(g, tb.idgen, s, {})
-        entries.add (globalName(s, g.config), tbody)
+        if wholeMode:
+          # Retain the transformed body on the sym so `writeSymDef` serializes it
+          # in the routine's `(sd)` 2-way-body slot.
+          s.transformedBody = tbody
+        else:
+          entries.add (globalName(s, g.config), tbody)
+        # Set `transformedBody` on nested ccClosure routines too (so a
+        # module-indexed nested closure gets its lifted body); in side-car mode
+        # this also appends their `(lowered)` entries.
         var seenNested = initIntSet()
         collectNestedClosureBodies(g, tb.idgen, tbody, s, seenNested, entries)
     # Collect the hooks this stage lifted, and transform each hook ROUTINE's body
@@ -546,13 +568,24 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
         # (`transformBody {}` returns the body but does not cache it).
         e.sym.transformedBody = transformBody(g, tb.idgen, e.sym, {})
       inc i
-    # Seal the index-loaded entities so their references in the bodies serialize
-    # as SymUses (resolved via the module index in cg), not duplicate defs.
-    sealLoadedBackendEntities(ast.program)
-    serializeLoweredBodies(g.config, modPos.int32, entries, hooks, artifact)
-    if isDefined(g.config, "icDceCheck"):
-      stderr.writeLine "[icLower] " & extractFilename(artifact) & " " &
-        $entries.len & " routines transformed, " & $hooks.len & " hooks"
+    if wholeMode:
+      # Re-serialize the whole module to its suffix-based `.t.nif` (the path
+      # `toNifFilename` resolves for the cg/emit stages). `writeLoweredModule`
+      # seals routines itself.
+      let suffix = cachedModuleSuffix(g.config, FileIndex modPos)
+      let wholeArtifact = toGeneratedFile(g.config, AbsoluteFile(suffix), ".t.nif").string
+      writeLoweredModule(ast.program, g.config, target, hooks, wholeArtifact)
+      if isDefined(g.config, "icDceCheck"):
+        stderr.writeLine "[icLowerWhole] " & extractFilename(wholeArtifact) & " " &
+          $hooks.len & " hooks"
+    else:
+      # Seal the index-loaded entities so their references in the bodies serialize
+      # as SymUses (resolved via the module index in cg), not duplicate defs.
+      sealLoadedBackendEntities(ast.program)
+      serializeLoweredBodies(g.config, modPos.int32, entries, hooks, artifact)
+      if isDefined(g.config, "icDceCheck"):
+        stderr.writeLine "[icLower] " & extractFilename(artifact) & " " &
+          $entries.len & " routines transformed, " & $hooks.len & " hooks"
   else:
     # DEFAULT (Stage 0, byte-neutral): record one empty-marker per owned routine.
     # `cg` derives the transformed body itself, so output is unchanged; this only
@@ -726,6 +759,7 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
   if precompSys.module != nil:
     replayBackendActions(g, precompSys.module, precompSys.topLevel)
   let bl = BModuleList(g.backend)
+  var addedCFiles = initHashSet[string]()
   for m in bl.mods:
     if m != nil:
       let cfile = getCFile(m)
@@ -733,10 +767,37 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
       # (extra members of system's closure that no build rule targets) had their
       # code emit-everywhere'd into the targets, so they have no file to compile.
       if not fileExists(cfile.string): continue
+      addedCFiles.incl extractFilename(cfile.string)
       var cf = Cfile(nimname: m.module.name.s, cname: cfile,
                      obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
                      flags: {})
       addFileToCompile(g.config, cf)
+  # deps.nim's static scanner can keep a CONDITIONALLY-imported module as a build
+  # node (e.g. `net`'s `when defineSsl: import openssl`, or a `when defined(os)`
+  # import) that the NIF-`deps` walk above never reaches because the condition is
+  # off. Such a node still emitted a `.c`, and it can OWN a live generic instance
+  # that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused by
+  # `strutils.escape`) — so its body must be at link or that reference is
+  # undefined. Link every emitted `.c` the merge decision says OWNS a LIVE symbol;
+  # a node that owns nothing live (a Windows-only winsock node on Linux) is
+  # correctly skipped. Whole-module only: the side-car default is unchanged.
+  if isDefined(g.config, "icWholeLowered"):
+    let nimcache = getNimcacheDir(g.config).string
+    let decision = readMergeDecision(nimcache / MergeDecisionFile)
+    if not decision.broken:
+      var liveOwners = initHashSet[string]()
+      for cname, owner in decision.owners:
+        if owner.endsWith(".c.nif") and cname in decision.live:
+          liveOwners.incl owner
+      for owner in liveOwners:
+        let cbase = owner[0 ..< owner.len - ".nif".len]  # "@m….nim.c.nif" -> ".c"
+        if addedCFiles.containsOrIncl(cbase): continue
+        let cfile = AbsoluteFile(nimcache / cbase)
+        if not fileExists(cfile.string): continue
+        var cf = Cfile(nimname: cbase, cname: cfile,
+                       obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
+                       flags: {})
+        addFileToCompile(g.config, cf)
   if g.config.cmd != cmdTcc:
     extccomp.callCCompiler(g.config)
 

compiler/options.nim

@@ -29,7 +29,7 @@ const
 
   nimEnableCovariance* = defined(nimEnableCovariance)
 
-  icFormatVersion* = "18"
+  icFormatVersion* = "20"
     ## Version of the IC cache format (the sem-NIF module layout written by
     ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
     ## that layout changes: `commandIc` wipes a nimcache whose `ic.version`