IC: wire the per-module backend into nifmake (Phase 2b, B5)

deps.nim's generateBackendBuildFile now emits the per-module pipeline instead of one whole-program nifc rule: per-module cg -> merge -> per-module emit -> link. The single nim_nifc command template carries the per-rule stage/module switches in nifmake's (args) slot; backendCFile reconstructs each module's .c path exactly as cgen.getCFile does (mangleModuleName of the source path for main, the NIF suffix for deps) so the rules can declare outputs without loading any backend module. The main module's cg depends on every other .c.nif (it reads their init metas for NimMain), so it runs last; merge depends on all .c.nif; each emit on merge; link on all .c. Supporting changes: - new `link` stage (nifbackend.generateLinkStage): registers every emitted .c and runs extccomp.callCCompiler once (parallel cc + link). Skips modules with no .c (extra members of system's closure whose code was emit-everywhere'd). - loadBackendModules also loads system's transitive closure so every module in the dep graph is a resolvable cg/emit target (was: project closure + system only, leaving system-closure modules unfindable). - cg always writes a .c.nif (even for code-less leaf modules) so every cg rule has its declared nifmake output. - export getSomeNameForModule; deps.nim imports modulepaths/extccomp/cnif. `nim ic` now builds via the per-module backend end to end. Validated: the int diamond and a generic+exception program build and run byte-correct vs the whole-program backend; koch ic thallo/tconverter/timp/tmiscs/tparseutils all green (and a thallo binary's output matches the whole-program build). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-14 15:43:45 +00:00 · 2026-06-13 23:17:37 +02:00
parent 8e0dd4bfb2
commit dcfa5573ca
4 changed files with 167 additions and 32 deletions
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -1864,7 +1864,7 @@ proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope =
  ## Returns a mangled module name.
  result = mangleModuleName(conf, filename).mangle

-proc getSomeNameForModule(m: BModule): Rope =
+proc getSomeNameForModule*(m: BModule): Rope =
  ## Returns a mangled module name.
  assert m.module.kind == skModule
  assert m.module.owner.kind == skPackage
--- a/compiler/deps.nim
+++ b/compiler/deps.nim
@@ -11,7 +11,8 @@
 ## This enables incremental and parallel compilation using the `m` switch.

 import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio]
-import options, msgs, lineinfos, pathutils, condsyms, icconfig
+import options, msgs, lineinfos, pathutils, condsyms, icconfig,
+  modulepaths, extccomp, cnif

 import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder]
 import "../dist/nimony/src/gear2" / modnames
@@ -866,61 +867,130 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin

  b.endTree()  # stmts

+proc backendCFile(c: DepContext; node: Node): string =
+  ## The `.c` path the backend writes for `node`, computed exactly as
+  ## `cgen.getCFile` does: `mangleModuleName` of the module's cfilename, which
+  ## is the source path for the main module (registered at its source index) and
+  ## the NIF suffix for every dependency (a `fikNifModule` whose `toFullPath` is
+  ## the suffix). Lets nifmake declare a per-module output without loading any
+  ## backend module.
+  let cfilename =
+    if node.id == 0: AbsoluteFile node.files[0].nimFile
+    else: AbsoluteFile node.files[0].modname
+  result = changeFileExt(completeCfilePath(c.config,
+    mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string
+
 proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
-  ## Backend build file: today a single whole-program `nim nifc` rule that reads
-  ## every module's semmed NIF and emits/compiles/links the executable. The
-  ## driver runs it once, after the frontend fixpoint (the semmed NIFs already
-  ## exist on disk, so they enter this graph as leaf inputs with no producing
-  ## rule, exactly like the nifler rules' source `.nim` inputs). The per-module
-  ## backend rewrite replaces this one rule with per-module codegen + a DCE rule
-  ## + a link rule; nothing else here changes.
+  ## Per-module backend build file. One `nim_nifc` command template (the actual
+  ## stage/module switches ride in each rule's `(args …)`), then the stages of
+  ## the per-module backend as separate nifmake rules:
+  ##   cg(per module) -> merge -> emit(per module) -> link
+  ## Every module's semmed NIF is a leaf input (produced by the frontend run).
+  ## `cg` emits a module's whole demanded closure into its `.c.nif`
+  ## (emit-everywhere); `merge` picks one owner per duplicated definition across
+  ## all `.c.nif`; `emit` renders each module's `.c` (dropping non-owned/dead
+  ## bodies); `link` compiles and links every `.c` in one `callCCompiler`. The
+  ## main module's `cg` depends on every other `.c.nif` because it reads their
+  ## init/datInit meta heads to wire up NimMain, so it must run last.
  let nimcache = getNimcacheDir(c.config).string
  createDir(nimcache)
  result = nimcache / c.nodes[0].files[0].modname & ".backend.build.nif"

+  let mainNif = c.nodes[0].files[0].nimFile
+  let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt)
+  let mergeFile = nimcache / MergeDecisionFile
+
+  # Per-node output paths.
+  var cnifFiles = newSeq[string](c.nodes.len)
+  var cFiles = newSeq[string](c.nodes.len)
+  for i, node in c.nodes:
+    cFiles[i] = backendCFile(c, node)
+    cnifFiles[i] = cFiles[i] & ".nif"
+
  var b = nifbuilder.open(result)
  defer: b.close()

  b.addHeader("nim ic", "nifmake")
  b.addTree "stmts"

-  # Define nim nifc command
+  # Command template: `nifc --nimcache … --path … <forwarded> <per-rule args>
+  # <project>`. The trailing `(args)` is filled per rule with the stage and
+  # module switches; `(input 0)` is the project file.
  b.addTree "cmd"
  b.addSymbolDef "nim_nifc"
  b.addStrLit getAppFilename()
  b.addStrLit "nifc"
  b.addStrLit "--nimcache:" & nimcache
-  # Add search paths
  for p in c.config.searchPaths:
    b.addStrLit "--path:" & p.string
  for a in forwardedArgs:
    b.addStrLit a
+  b.addTree "args"
+  b.endTree()
  b.addTree "input"
  b.addIntLit 0
  b.endTree()
  b.endTree()

-  # Final compilation step: generate executable from main module
-  let mainNif = c.nodes[0].files[0].nimFile
-  let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt)
+  template inputStr(s: string) =
+    b.addTree "input"
+    b.addStrLit s
+    b.endTree()
+  template outputStr(s: string) =
+    b.addTree "output"
+    b.addStrLit s
+    b.endTree()
+
+  # cg: one rule per module. Inputs are the project (slot 0) and every semmed
+  # NIF (so the whole program loads and the rule is ordered after the frontend);
+  # the main module additionally depends on every other `.c.nif` (init metas).
+  for i, node in c.nodes:
+    b.addTree "do"
+    b.addIdent "nim_nifc"
+    b.withTree "args":
+      b.addStrLit "--icBackendStage:cg"
+      b.addStrLit "--icBackendModule:" & node.files[0].modname
+    inputStr mainNif
+    for n2 in c.nodes:
+      inputStr c.semmedFile(n2.files[0])
+    if node.id == 0:
+      for j in 0 ..< c.nodes.len:
+        if c.nodes[j].id != 0:
+          inputStr cnifFiles[j]
+    outputStr cnifFiles[i]
+    b.endTree()
+
+  # merge: read every `.c.nif`, write the ownership/liveness decision.
  b.addTree "do"
  b.addIdent "nim_nifc"
-  # Input: .nim file (expanded as argument)
-  b.addTree "input"
-  b.addStrLit mainNif
+  b.withTree "args":
+    b.addStrLit "--icBackendStage:merge"
+  inputStr mainNif
+  for cn in cnifFiles: inputStr cn
+  outputStr mergeFile
  b.endTree()
-  # Also depend on the semmed .nif files of the main module and all its
-  # dependencies. nifmake's topological sort orders nodes by depth; without
-  # these inputs the nim_nifc node sits at depth 1 (no recognized inputs)
-  # alongside the nifler nodes and runs *before* the nim_m steps that
-  # produce the .nif files it needs to read.
-  for node in c.nodes:
-    b.addTree "input"
-    b.addStrLit c.semmedFile(node.files[0])
+
+  # emit: render each module's `.c` from its `.c.nif` + the merge decision.
+  for i, node in c.nodes:
+    b.addTree "do"
+    b.addIdent "nim_nifc"
+    b.withTree "args":
+      b.addStrLit "--icBackendStage:emit"
+      b.addStrLit "--icBackendModule:" & node.files[0].modname
+    inputStr mainNif
+    inputStr cnifFiles[i]
+    inputStr mergeFile
+    outputStr cFiles[i]
    b.endTree()
-  b.addTree "output"
-  b.addStrLit exeFile
-  b.endTree()
+
+  # link: compile + link every emitted `.c` in one process.
+  b.addTree "do"
+  b.addIdent "nim_nifc"
+  b.withTree "args":
+    b.addStrLit "--icBackendStage:link"
+  inputStr mainNif
+  for cf in cFiles: inputStr cf
+  outputStr exeFile
  b.endTree()

  b.endTree()  # stmts
--- a/compiler/nifbackend.nim
+++ b/compiler/nifbackend.nim
@@ -520,7 +520,30 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
  var precompSys = moduleFromNifFile(g, systemFileIdx, {LoadFullAst, AlwaysLoadInterface})
  g.systemModule = precompSys.module
  var nifFiles: seq[string] = @[toNifFilename(g.config, systemFileIdx)]
-  let modules = loadModuleDependencies(g, mainFileIdx, nifFiles)
+  var modules = loadModuleDependencies(g, mainFileIdx, nifFiles)
+  # loadModuleDependencies traverses the project's import closure and stops at
+  # system. The whole-program backend then demand-loads system's own closure
+  # (locks, allocators, threads, …) during codegen; the per-module backend
+  # instead makes every one of those a first-class cg/emit target, so load that
+  # closure here too — otherwise `findTargetModule` cannot resolve their suffix.
+  block:
+    var visited = initHashSet[string]()
+    visited.incl "sysma2dyk"
+    for m in modules:
+      visited.incl cachedModuleSuffix(g.config, FileIndex m.module.position)
+    var stack: seq[ModuleSuffix] = @[]
+    if precompSys.module != nil:
+      for dep in precompSys.deps: stack.add dep
+    while stack.len > 0:
+      let suffix = stack.pop()
+      if not visited.containsOrIncl(suffix.string):
+        var isKnown = false
+        let fileIdx = registerNifSuffix(g.config, suffix.string, isKnown)
+        let precomp = moduleFromNifFile(g, fileIdx, {LoadFullAst})
+        if precomp.module != nil:
+          modules.add precomp
+          nifFiles.add toNifFilename(g.config, fileIdx)
+          for dep in precomp.deps: stack.add dep
  flushMethodReplays(g)
  for m in modules:
    discard setupNifBackendModule(g, m.module)
@@ -590,6 +613,17 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
  # so `cgenWriteModules` emits no artifact for it). cc/link are NOT run here.
  cgenWriteModules(g.backend, g.config)

+  # Always leave a `.c.nif` for the target, even when the module has no code
+  # (a leaf library whose procs all emit into their users): the per-module
+  # nifmake graph declares one `.c.nif` output per `cg` rule, so a missing one
+  # would re-fire the rule forever. An empty artifact renders to an empty `.c`.
+  if tb != nil:
+    let artifact = getCFile(tb).string & ".nif"
+    if not fileExists(artifact):
+      writeCnifArtifact("", artifact,
+        semmedNif = toNifFilename(g.config, FileIndex target.module.position),
+        moduleBase = $getSomeNameForModule(tb))
+
 proc generateMergeStage(g: ModuleGraph) =
  ## Per-module backend merge (`--icBackendStage:merge`): a pure artifact
  ## operation, no module graph loaded. Reads every `.c.nif` the `cg` stages
@@ -645,6 +679,32 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
    stderr.writeLine "[icEmit] " & extractFilename(cfile) & " dropped " &
      $dropped & " bodies (" & $code.len & " bytes)"

+proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
+  ## Per-module backend link (`--icBackendStage:link`): the `emit` stages have
+  ## written every module's `.c`; register them and run the C compiler + linker
+  ## once via `extccomp.callCCompiler` (which parallelizes the per-file cc and
+  ## skips up-to-date objects itself). No codegen runs — the graph is loaded only
+  ## so `getCFile` yields each module's emitted `.c` path.
+  let (modules, precompSys, _) = loadBackendModules(g, mainFileIdx)
+  if modules.len == 0:
+    rawMessage(g.config, errGenerated,
+      "Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
+    return
+  let bl = BModuleList(g.backend)
+  for m in bl.mods:
+    if m != nil:
+      let cfile = getCFile(m)
+      # Only modules that are their own cg/emit target produced a `.c`; the rest
+      # (extra members of system's closure that no build rule targets) had their
+      # code emit-everywhere'd into the targets, so they have no file to compile.
+      if not fileExists(cfile.string): continue
+      var cf = Cfile(nimname: m.module.name.s, cname: cfile,
+                     obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
+                     flags: {})
+      addFileToCompile(g.config, cf)
+  if g.config.cmd != cmdTcc:
+    extccomp.callCCompiler(g.config)
+
 proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
  ## Main entry point for NIF-based C code generation.
  ## Traverses the module dependency graph and generates C code.
@@ -657,6 +717,9 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
  elif g.config.icBackendStage == "emit":
    generateEmitStage(g, mainFileIdx)
    return
+  elif g.config.icBackendStage == "link":
+    generateLinkStage(g, mainFileIdx)
+    return
  elif g.config.icBackendStage.len > 0:
    rawMessage(g.config, errGenerated,
      "per-module backend stage not implemented yet: " & g.config.icBackendStage)
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -430,9 +430,11 @@ type
                              # module to its `.c.nif`), "merge" (global liveness
                              # + owner assignment across all `.c.nif`), "emit"
                              # (render one module's `.c` from its `.c.nif` + the
-                              # merge decision). Empty = today's whole-program
-                              # backend (load all, codegen+DCE+cc+link in one
-                              # process). See `compiler/nifbackend.nim`.
+                              # merge decision), "link" (cc + link every emitted
+                              # `.c`). Empty = whole-program backend (load all,
+                              # codegen+DCE+cc+link in one process). The stages
+                              # are wired as nifmake rules by `deps.nim`'s backend
+                              # build file. See `compiler/nifbackend.nim`.
    icBackendModule*: string  # under `nim nifc` with icBackendStage in {cg,emit}:
                              # the NIF module suffix this invocation codegens or
                              # emits. The other modules are loaded only so types