IC: codegen progress

2026-08-22 23:00:58 +00:00 · 2026-06-25 09:19:52 +02:00
parent e75690f414
commit 4a9424b682
4 changed files with 159 additions and 20 deletions
--- a/compiler/ccgthreadvars.nim
+++ b/compiler/ccgthreadvars.nim
@@ -39,11 +39,30 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
      if isExtern: Extern
      elif lfExportLib in s.loc.flags: ExportLibVar
      else: Private
-    m.s[cfsVars].addVar(m, s,
-      name = s.loc.snippet,
-      typ = getTypeDesc(m, s.loc.t),
-      kind = Threadvar,
-      visibility = vis)
+    if m.config.cmd == cmdNifC and vis == Private and not isExtern:
+      # A `{.threadvar.}`/`{.global.}` thread-local declared inside a routine is
+      # emitted by every module that emit-everywhere's its enclosing routine
+      # (e.g. libp2p's `var keys {.global.}: HashSet`), so its content-addressed
+      # name collides at link. Same fix as a plain global (genGlobalVarDecl):
+      # `extern` declaration + a droppable `'d'` definition unit the merge stage
+      # assigns one owner. The thread-local storage class rides on both.
+      let cname = stripCnifMarks(s.loc.snippet)
+      let td = getTypeDesc(m, s.loc.t)
+      # `extern` declaration via the full `addVar` overload — it knows the
+      # thread-local storage class (`NIM_THREADVAR`); the simple `addVar`'s
+      # `addVarHeader` does not implement `Threadvar`.
+      m.s[cfsVars].addVar(m, s, name = s.loc.snippet, typ = td,
+                          kind = Threadvar, visibility = Extern)
+      m.s[cfsVars].add(cnifDefDirective(cname, "d", icNifName(m, s)))
+      m.s[cfsVars].addVar(m, s,
+        name = s.loc.snippet, typ = td, kind = Threadvar, visibility = vis)
+      m.s[cfsVars].add(cnifEndDefs())
+    else:
+      m.s[cfsVars].addVar(m, s,
+        name = s.loc.snippet,
+        typ = getTypeDesc(m, s.loc.t),
+        kind = Threadvar,
+        visibility = vis)

 proc generateThreadLocalStorage(m: BModule) =
  if m.g.nimtv.buf.len != 0 and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
--- a/compiler/ccgtypes.nim
+++ b/compiler/ccgtypes.nim
@@ -1518,8 +1518,25 @@ proc genObjectFields(m: BModule; typ, origType: PType, n: PNode, expr: Rope;
    m.s[cfsTypeInit3].addFieldAssignment(expr, "name", makeCString(field.name.s))
    m.s[cfsTypeInit3].addFieldAssignment(expr, "sons", cAddr(subscript(tmp, cIntValue(0))))
    m.s[cfsTypeInit3].addFieldAssignment(expr, "len", L)
-    m.s[cfsData].addArrayVar(kind = Local, name = tmp,
-      elementType = ptrType("TNimNode"), len = toInt(L)+1)
+    if m.config.cmd == cmdNifC:
+      # The discriminator table has a content-addressed name
+      # (`NimDT_<hashType>_<field>`) and is emitted by every module that demands
+      # this variant type's RTTI (emit-everywhere; RTTI has no single owner —
+      # emission is lazy and often skipped). Declare it `extern` + wrap the
+      # tentative definition as a droppable `'d'` unit so the merge stage keeps
+      # exactly one external-linkage definition (mirrors the `TNimType` var and
+      # consts); otherwise the identical name collides across modules at link.
+      m.s[cfsData].addDeclWithVisibility(Extern):
+        m.s[cfsData].addArrayVar(kind = Local, name = tmp,
+          elementType = ptrType("TNimNode"), len = toInt(L)+1)
+      m.s[cfsData].add(cnifDefDirective(tmp, "d", ""))
+      m.s[cfsData].addArrayVar(kind = Local, name = tmp,
+        elementType = ptrType("TNimNode"), len = toInt(L)+1)
+      m.s[cfsData].add(cnifEndDefs())
+      m.icDataDefs.add (tmp, "")
+    else:
+      m.s[cfsData].addArrayVar(kind = Local, name = tmp,
+        elementType = ptrType("TNimNode"), len = toInt(L)+1)
    for i in 1..<n.len:
      var b = n[i]           # branch
      var tmp2 = getNimNode(m)
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -125,10 +125,25 @@ proc emitsBodyInThisModule(m: BModule, prc: PSym): bool =
  ## Generic instances and synthesized hooks (`=destroy`, `$`, …) have no single
  ## owning-module top-level — they are minted on demand — so each demander emits
  ## them and the merge stage deduplicates by their content-addressed C name.
+  ##
+  ## A NESTED routine is not emitted on its own: it is lambda-lifted and emitted
+  ## as part of its ENCLOSING routine's body, into the same TU. So the decision
+  ## must follow the OUTERMOST enclosing routine (the one directly under the
+  ## module — `skipGenericOwner` stops at a generic *instance*, not its
+  ## originating generic), never the nested symbol's own identity. Otherwise a
+  ## nested proc whose enclosing is a generic instance (content-addressed,
+  ## emitted by every demander) — e.g. nim-serialization's per-field `readField`
+  ## inside the `makeFieldReadersTable[R,W]` instance, whose address fills the
+  ## returned table — is gated out (its own `itemId.module` is the minting module
+  ## and its disamb is a plain counter), so the enclosing's lift degrades it to a
+  ## prototype and its body lands in no TU → undefined at link.
  if not (m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"):
    return true
-  result = prc.itemId.module == m.module.position or
-           (prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32
+  var top = prc
+  while top.skipGenericOwner != nil and top.skipGenericOwner.kind != skModule:
+    top = top.skipGenericOwner
+  result = top.itemId.module == m.module.position or
+           (top.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32

 proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc =
  result = TLoc(k: k, storage: s, lode: lode,
@@ -776,12 +791,31 @@ proc genGlobalVarDecl(res: var Builder, p: BProc, n: PNode; td: Snippet;
    typ = constType(typ)
  if p.hcrOn:
    typ = ptrType(typ)
-  res.addVar(p.module, s,
-    name = s.loc.snippet,
-    typ = typ,
-    visibility = vis,
-    initializer = initializer,
-    initializerKind = initializerKind)
+  if p.config.cmd == cmdNifC and vis == Private and sfImportc notin s.flags:
+    # A `{.global.}` var (e.g. chronos's per-call-site `var loc {.global.} =
+    # SrcLoc(...)`, or a gensym'd `var dummy`/`var topic` with no initializer)
+    # declared inside a routine is emitted by every module that emit-everywhere's
+    # its enclosing routine; its content-addressed name then collides at link.
+    # Declare it `extern` + wrap the definition as a droppable `'d'` unit so the
+    # merge stage keeps exactly one (like consts / TNimType / the NimDT
+    # discriminator tables / the threadvar path). This covers no-initializer
+    # globals too — they collide just the same. A module-level global has a
+    # single claimant → its sole emitter is the owner merge keeps.
+    let cname = stripCnifMarks(s.loc.snippet)
+    res.addDeclWithVisibility(Extern):
+      res.addVar(kind = Local, name = s.loc.snippet, typ = typ)
+    res.add(cnifDefDirective(cname, "d", icNifName(p.module, s)))
+    res.addVar(p.module, s,
+      name = s.loc.snippet, typ = typ, visibility = vis,
+      initializer = initializer, initializerKind = initializerKind)
+    res.add(cnifEndDefs())
+  else:
+    res.addVar(p.module, s,
+      name = s.loc.snippet,
+      typ = typ,
+      visibility = vis,
+      initializer = initializer,
+      initializerKind = initializerKind)

 proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
  let s = n.sym
--- a/compiler/deps.nim
+++ b/compiler/deps.nim
@@ -918,6 +918,36 @@ proc backendCFile(c: DepContext; node: Node): string =
  result = changeFileExt(completeCfilePath(c.config,
    mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string

+proc computeLiveBackendNodes(c: DepContext): seq[bool] =
+  ## Which nodes the backend must code-generate: the closure reachable from the
+  ## program roots (main + `system` + `--import`ed modules) via the REAL,
+  ## post-sem import edges (`.s.deps`).
+  ##
+  ## The static `.deps` scan over-approximates: it cannot evaluate guards like
+  ## `when defined(windows)` or const-aliased ones (`when useWinVersion`, with
+  ## `const useWinVersion = defined(windows) or defined(nimdoc)`), so it keeps
+  ## the dead branch's import. e.g. on Linux `nativesockets`'s static deps list
+  ## `winlean`; the discovery fixpoint only ever *adds* edges, never prunes, so
+  ## `winlean` stays a node and got a full `lower`/`cg`/`emit`/link pipeline.
+  ## That is harmless for sem (an extra `nim m`) but fatal for codegen:
+  ## `winlean`'s `importc, header: "winsock2.h"` decls emit
+  ## `#include "winsock2.h"` into a C file that cannot compile off-Windows.
+  ## Sem's resolved import set (`.s.deps`) is the real program graph — the
+  ## non-IC compiler would never touch `winlean` here — so restrict the backend
+  ## to it. (`.s.deps` is the same data the discovery loop trusts; it is written
+  ## for every sem'd module, including grouped SCC members.)
+  result = newSeq[bool](c.nodes.len)
+  var stack: seq[int] = @[0]            # main module
+  if c.systemNodeId >= 0: stack.add c.systemNodeId
+  for impId in c.implicitNodeIds: stack.add impId  # every module imports these
+  while stack.len > 0:
+    let ni = stack.pop()
+    if ni < 0 or ni >= c.nodes.len or result[ni]: continue
+    result[ni] = true
+    for p in readSemDeps(c, c.nodes[ni].files[0]):
+      let idx = c.processedModules.getOrDefault(c.toPair(p).modname, -1)
+      if idx >= 0: stack.add idx
+
 proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
  ## Per-module backend build file. One `nim_nifc` command template (the actual
  ## stage/module switches ride in each rule's `(args …)`), then the stages of
@@ -950,6 +980,34 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
    cnifFiles[i] = cFiles[i] & ".nif"
    tFiles[i] = nimcache / node.files[0].modname & ".t.nif"

+  # Only code-generate modules the real program actually reaches; statically
+  # over-approximated nodes (e.g. `winlean` on Linux) are sem'd but not emitted.
+  let live = computeLiveBackendNodes(c)
+  # Drop a pruned node's stale backend artifacts: the `merge` stage globs
+  # `*.c.nif` off disk (not the build-file inputs) and the `link` stage scans
+  # the loaded closure's `.c`s, so a leftover `.c.nif`/`.c` from a run before
+  # this module became unreachable (a prior over-approximated build, or an edit
+  # that removed its last real importer) would still be merged/compiled —
+  # reintroducing exactly the off-platform `#include` this prune avoids.
+  var prunedStale = false
+  for i in 0 ..< c.nodes.len:
+    if not live[i]:
+      # `fileExists` before remove so we only force a merge recompute (below)
+      # when an artifact was actually present — i.e. a build where this module
+      # WAS emitted, not the steady state where it never is.
+      if fileExists(cnifFiles[i]) or fileExists(cFiles[i]): prunedStale = true
+      removeFile(cnifFiles[i])
+      removeFile(cFiles[i])
+  # The merge decision is a pure function of the set of `.c.nif`s present; if we
+  # just removed an over-approximated module's artifacts, a decision computed
+  # while they were present is stale — it can name a now-absent module as a
+  # symbol's owner (`asyncdispatch` owning `NTIdomain` here), leaving that symbol
+  # undefined at link. nifmake will not re-fire `merge` on its own: dropping an
+  # input makes no remaining input newer than the output. Delete the decision so
+  # the (now missing) output forces a recompute against the live `.c.nif` set.
+  if prunedStale:
+    removeFile(mergeFile)
+
  var b = nifbuilder.open(result)
  defer: b.close()

@@ -1000,6 +1058,7 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
  # dependency re-sems (and re-emits the `.s.nif` of) every transitive importer;
  # a module whose own `.s.nif` is unchanged genuinely needs no re-lowering.
  for i, node in c.nodes:
+    if not live[i]: continue
    b.addTree "do"
    b.addIdent "nim_nifc"
    b.withTree "args":
@@ -1020,6 +1079,7 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
  # stale copy here is harmless. The main module additionally depends on every
  # other `.c.nif` (it reads their init/datInit metas to wire up NimMain).
  for i, node in c.nodes:
+    if not live[i]: continue
    b.addTree "do"
    b.addIdent "nim_nifc"
    b.withTree "args":
@@ -1028,7 +1088,7 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
    inputStr tFiles[i]
    if node.id == 0:
      for j in 0 ..< c.nodes.len:
-        if c.nodes[j].id != 0:
+        if c.nodes[j].id != 0 and live[j]:
          inputStr cnifFiles[j]
    outputStr cnifFiles[i]
    b.endTree()
@@ -1038,12 +1098,14 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
  b.addIdent "nim_nifc"
  b.withTree "args":
    b.addStrLit "--icBackendStage:merge"
-  for cn in cnifFiles: inputStr cn
+  for i in 0 ..< c.nodes.len:
+    if live[i]: inputStr cnifFiles[i]
  outputStr mergeFile
  b.endTree()

  # emit: render each module's `.c` from its `.c.nif` + the merge decision.
  for i, node in c.nodes:
+    if not live[i]: continue
    b.addTree "do"
    b.addIdent "nim_nifc"
    b.withTree "args":
@@ -1064,7 +1126,8 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
  b.addIdent "nim_nifc"
  b.withTree "args":
    b.addStrLit "--icBackendStage:link"
-  for cf in cFiles: inputStr cf
+  for i in 0 ..< c.nodes.len:
+    if live[i]: inputStr cFiles[i]
  outputStr exeFile
  b.endTree()

@@ -1187,8 +1250,14 @@ proc commandIc*(conf: ConfigRef) =
    # each DAG depth via execProcesses (defaults to all cores). Cold builds are
    # otherwise serial (one child at a time) and leave the machine idle. Opt out
    # with `-d:icNoParallel` (e.g. for readable, non-interleaved child output
-    # when debugging a build).
-    let parallel = if isDefined(conf, "icNoParallel"): "" else: " --parallel"
+    # when debugging a build), or cap the concurrency with `-d:icJobs:N` — an
+    # uncapped fan-out across many cores can exhaust RAM on a large project
+    # (each `nim m`/`cg` child holds its own module graph), which nifmake's own
+    # `-j:N` exists to bound.
+    let parallel =
+      if isDefined(conf, "icNoParallel"): ""
+      elif isDefined(conf, "icJobs"): " --parallel:" & conf.symbols["icJobs"]
+      else: " --parallel"

    # Phase 1 — frontend (nifler + `nim m`), run to a discovery fixpoint.
    var rounds = 0