diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 8013cfb2be..02fcdf8ecb 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1864,7 +1864,7 @@ proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope = ## Returns a mangled module name. result = mangleModuleName(conf, filename).mangle -proc getSomeNameForModule(m: BModule): Rope = +proc getSomeNameForModule*(m: BModule): Rope = ## Returns a mangled module name. assert m.module.kind == skModule assert m.module.owner.kind == skPackage diff --git a/compiler/deps.nim b/compiler/deps.nim index 37b014b35e..07e4652673 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -11,7 +11,8 @@ ## This enables incremental and parallel compilation using the `m` switch. import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio] -import options, msgs, lineinfos, pathutils, condsyms, icconfig +import options, msgs, lineinfos, pathutils, condsyms, icconfig, + modulepaths, extccomp, cnif import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder] import "../dist/nimony/src/gear2" / modnames @@ -866,61 +867,130 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin b.endTree() # stmts +proc backendCFile(c: DepContext; node: Node): string = + ## The `.c` path the backend writes for `node`, computed exactly as + ## `cgen.getCFile` does: `mangleModuleName` of the module's cfilename, which + ## is the source path for the main module (registered at its source index) and + ## the NIF suffix for every dependency (a `fikNifModule` whose `toFullPath` is + ## the suffix). Lets nifmake declare a per-module output without loading any + ## backend module. + let cfilename = + if node.id == 0: AbsoluteFile node.files[0].nimFile + else: AbsoluteFile node.files[0].modname + result = changeFileExt(completeCfilePath(c.config, + mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string + proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string = - ## Backend build file: today a single whole-program `nim nifc` rule that reads - ## every module's semmed NIF and emits/compiles/links the executable. The - ## driver runs it once, after the frontend fixpoint (the semmed NIFs already - ## exist on disk, so they enter this graph as leaf inputs with no producing - ## rule, exactly like the nifler rules' source `.nim` inputs). The per-module - ## backend rewrite replaces this one rule with per-module codegen + a DCE rule - ## + a link rule; nothing else here changes. + ## Per-module backend build file. One `nim_nifc` command template (the actual + ## stage/module switches ride in each rule's `(args …)`), then the stages of + ## the per-module backend as separate nifmake rules: + ## cg(per module) -> merge -> emit(per module) -> link + ## Every module's semmed NIF is a leaf input (produced by the frontend run). + ## `cg` emits a module's whole demanded closure into its `.c.nif` + ## (emit-everywhere); `merge` picks one owner per duplicated definition across + ## all `.c.nif`; `emit` renders each module's `.c` (dropping non-owned/dead + ## bodies); `link` compiles and links every `.c` in one `callCCompiler`. The + ## main module's `cg` depends on every other `.c.nif` because it reads their + ## init/datInit meta heads to wire up NimMain, so it must run last. let nimcache = getNimcacheDir(c.config).string createDir(nimcache) result = nimcache / c.nodes[0].files[0].modname & ".backend.build.nif" + let mainNif = c.nodes[0].files[0].nimFile + let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt) + let mergeFile = nimcache / MergeDecisionFile + + # Per-node output paths. + var cnifFiles = newSeq[string](c.nodes.len) + var cFiles = newSeq[string](c.nodes.len) + for i, node in c.nodes: + cFiles[i] = backendCFile(c, node) + cnifFiles[i] = cFiles[i] & ".nif" + var b = nifbuilder.open(result) defer: b.close() b.addHeader("nim ic", "nifmake") b.addTree "stmts" - # Define nim nifc command + # Command template: `nifc --nimcache … --path … + # `. The trailing `(args)` is filled per rule with the stage and + # module switches; `(input 0)` is the project file. b.addTree "cmd" b.addSymbolDef "nim_nifc" b.addStrLit getAppFilename() b.addStrLit "nifc" b.addStrLit "--nimcache:" & nimcache - # Add search paths for p in c.config.searchPaths: b.addStrLit "--path:" & p.string for a in forwardedArgs: b.addStrLit a + b.addTree "args" + b.endTree() b.addTree "input" b.addIntLit 0 b.endTree() b.endTree() - # Final compilation step: generate executable from main module - let mainNif = c.nodes[0].files[0].nimFile - let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt) + template inputStr(s: string) = + b.addTree "input" + b.addStrLit s + b.endTree() + template outputStr(s: string) = + b.addTree "output" + b.addStrLit s + b.endTree() + + # cg: one rule per module. Inputs are the project (slot 0) and every semmed + # NIF (so the whole program loads and the rule is ordered after the frontend); + # the main module additionally depends on every other `.c.nif` (init metas). + for i, node in c.nodes: + b.addTree "do" + b.addIdent "nim_nifc" + b.withTree "args": + b.addStrLit "--icBackendStage:cg" + b.addStrLit "--icBackendModule:" & node.files[0].modname + inputStr mainNif + for n2 in c.nodes: + inputStr c.semmedFile(n2.files[0]) + if node.id == 0: + for j in 0 ..< c.nodes.len: + if c.nodes[j].id != 0: + inputStr cnifFiles[j] + outputStr cnifFiles[i] + b.endTree() + + # merge: read every `.c.nif`, write the ownership/liveness decision. b.addTree "do" b.addIdent "nim_nifc" - # Input: .nim file (expanded as argument) - b.addTree "input" - b.addStrLit mainNif + b.withTree "args": + b.addStrLit "--icBackendStage:merge" + inputStr mainNif + for cn in cnifFiles: inputStr cn + outputStr mergeFile b.endTree() - # Also depend on the semmed .nif files of the main module and all its - # dependencies. nifmake's topological sort orders nodes by depth; without - # these inputs the nim_nifc node sits at depth 1 (no recognized inputs) - # alongside the nifler nodes and runs *before* the nim_m steps that - # produce the .nif files it needs to read. - for node in c.nodes: - b.addTree "input" - b.addStrLit c.semmedFile(node.files[0]) + + # emit: render each module's `.c` from its `.c.nif` + the merge decision. + for i, node in c.nodes: + b.addTree "do" + b.addIdent "nim_nifc" + b.withTree "args": + b.addStrLit "--icBackendStage:emit" + b.addStrLit "--icBackendModule:" & node.files[0].modname + inputStr mainNif + inputStr cnifFiles[i] + inputStr mergeFile + outputStr cFiles[i] b.endTree() - b.addTree "output" - b.addStrLit exeFile - b.endTree() + + # link: compile + link every emitted `.c` in one process. + b.addTree "do" + b.addIdent "nim_nifc" + b.withTree "args": + b.addStrLit "--icBackendStage:link" + inputStr mainNif + for cf in cFiles: inputStr cf + outputStr exeFile b.endTree() b.endTree() # stmts diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index 9514a4aa79..0037baf03c 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -520,7 +520,30 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex): var precompSys = moduleFromNifFile(g, systemFileIdx, {LoadFullAst, AlwaysLoadInterface}) g.systemModule = precompSys.module var nifFiles: seq[string] = @[toNifFilename(g.config, systemFileIdx)] - let modules = loadModuleDependencies(g, mainFileIdx, nifFiles) + var modules = loadModuleDependencies(g, mainFileIdx, nifFiles) + # loadModuleDependencies traverses the project's import closure and stops at + # system. The whole-program backend then demand-loads system's own closure + # (locks, allocators, threads, …) during codegen; the per-module backend + # instead makes every one of those a first-class cg/emit target, so load that + # closure here too — otherwise `findTargetModule` cannot resolve their suffix. + block: + var visited = initHashSet[string]() + visited.incl "sysma2dyk" + for m in modules: + visited.incl cachedModuleSuffix(g.config, FileIndex m.module.position) + var stack: seq[ModuleSuffix] = @[] + if precompSys.module != nil: + for dep in precompSys.deps: stack.add dep + while stack.len > 0: + let suffix = stack.pop() + if not visited.containsOrIncl(suffix.string): + var isKnown = false + let fileIdx = registerNifSuffix(g.config, suffix.string, isKnown) + let precomp = moduleFromNifFile(g, fileIdx, {LoadFullAst}) + if precomp.module != nil: + modules.add precomp + nifFiles.add toNifFilename(g.config, fileIdx) + for dep in precomp.deps: stack.add dep flushMethodReplays(g) for m in modules: discard setupNifBackendModule(g, m.module) @@ -590,6 +613,17 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = # so `cgenWriteModules` emits no artifact for it). cc/link are NOT run here. cgenWriteModules(g.backend, g.config) + # Always leave a `.c.nif` for the target, even when the module has no code + # (a leaf library whose procs all emit into their users): the per-module + # nifmake graph declares one `.c.nif` output per `cg` rule, so a missing one + # would re-fire the rule forever. An empty artifact renders to an empty `.c`. + if tb != nil: + let artifact = getCFile(tb).string & ".nif" + if not fileExists(artifact): + writeCnifArtifact("", artifact, + semmedNif = toNifFilename(g.config, FileIndex target.module.position), + moduleBase = $getSomeNameForModule(tb)) + proc generateMergeStage(g: ModuleGraph) = ## Per-module backend merge (`--icBackendStage:merge`): a pure artifact ## operation, no module graph loaded. Reads every `.c.nif` the `cg` stages @@ -645,6 +679,32 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) = stderr.writeLine "[icEmit] " & extractFilename(cfile) & " dropped " & $dropped & " bodies (" & $code.len & " bytes)" +proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) = + ## Per-module backend link (`--icBackendStage:link`): the `emit` stages have + ## written every module's `.c`; register them and run the C compiler + linker + ## once via `extccomp.callCCompiler` (which parallelizes the per-file cc and + ## skips up-to-date objects itself). No codegen runs — the graph is loaded only + ## so `getCFile` yields each module's emitted `.c` path. + let (modules, precompSys, _) = loadBackendModules(g, mainFileIdx) + if modules.len == 0: + rawMessage(g.config, errGenerated, + "Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx)) + return + let bl = BModuleList(g.backend) + for m in bl.mods: + if m != nil: + let cfile = getCFile(m) + # Only modules that are their own cg/emit target produced a `.c`; the rest + # (extra members of system's closure that no build rule targets) had their + # code emit-everywhere'd into the targets, so they have no file to compile. + if not fileExists(cfile.string): continue + var cf = Cfile(nimname: m.module.name.s, cname: cfile, + obj: completeCfilePath(g.config, toObjFile(g.config, cfile)), + flags: {}) + addFileToCompile(g.config, cf) + if g.config.cmd != cmdTcc: + extccomp.callCCompiler(g.config) + proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = ## Main entry point for NIF-based C code generation. ## Traverses the module dependency graph and generates C code. @@ -657,6 +717,9 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = elif g.config.icBackendStage == "emit": generateEmitStage(g, mainFileIdx) return + elif g.config.icBackendStage == "link": + generateLinkStage(g, mainFileIdx) + return elif g.config.icBackendStage.len > 0: rawMessage(g.config, errGenerated, "per-module backend stage not implemented yet: " & g.config.icBackendStage) diff --git a/compiler/options.nim b/compiler/options.nim index 06c05a94e0..f1b8ee7cb4 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -430,9 +430,11 @@ type # module to its `.c.nif`), "merge" (global liveness # + owner assignment across all `.c.nif`), "emit" # (render one module's `.c` from its `.c.nif` + the - # merge decision). Empty = today's whole-program - # backend (load all, codegen+DCE+cc+link in one - # process). See `compiler/nifbackend.nim`. + # merge decision), "link" (cc + link every emitted + # `.c`). Empty = whole-program backend (load all, + # codegen+DCE+cc+link in one process). The stages + # are wired as nifmake rules by `deps.nim`'s backend + # build file. See `compiler/nifbackend.nim`. icBackendModule*: string # under `nim nifc` with icBackendStage in {cg,emit}: # the NIF module suffix this invocation codegens or # emits. The other modules are loaded only so types