From 051ac6796897ab1ecae618b1f20c2ea230514f55 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 13 Jun 2026 20:50:11 +0200 Subject: [PATCH] IC: split the build file into frontend and backend (Phase 2a) generateBuildFile becomes three procs: computeForwardedArgs (config/define forwarding + writeIcConfig, depends only on config so computed once), generateFrontendBuildFile (nifler + nim m rules), and generateBackendBuildFile (today's single whole-program nim nifc rule; semmed NIFs enter as leaf inputs with no producing rule, like nifler's .nim source inputs). commandIc now runs two nifmake passes: phase 1 drives the frontend to the .s.deps discovery fixpoint, phase 2 runs the backend once over the now-final graph. Backend rebuilds are then a pure nifmake mtime decision, independent of frontend discovery -- and the backend file is the slot the per-module codegen + DCE + link rules drop into next. Observably inert: koch boot and koch bootic both reach their byte-identical fixed points (clean ric_ cache), 2-module cold/warm/body-edit correct, koch ic thallo green. Co-Authored-By: Claude Opus 4.8 --- compiler/deps.nim | 209 +++++++++++++++++++++++++++++----------------- 1 file changed, 132 insertions(+), 77 deletions(-) diff --git a/compiler/deps.nim b/compiler/deps.nim index 2cb48febda..37b014b35e 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -626,18 +626,12 @@ proc computeSCCs(c: DepContext): seq[seq[int]] = if work.len > 0: lowlink[work[^1].v] = min(lowlink[work[^1].v], lowlink[v]) -proc generateBuildFile(c: DepContext): string = - ## Generate the .build.nif file for nifmake - let nimcache = getNimcacheDir(c.config).string - createDir(nimcache) - result = nimcache / c.nodes[0].files[0].modname & ".build.nif" - - var b = nifbuilder.open(result) - defer: b.close() - - b.addHeader("nim ic", "nifmake") - b.addTree "stmts" - +proc computeForwardedArgs(c: DepContext): seq[string] = + ## Config/define forwarding shared by the frontend (`nim m`) and backend + ## (`nim nifc`) child commands. Depends only on the driver's config, not on + ## the dependency graph, so it is computed once per `nim ic` run (and also + ## writes the precompiled-config artifact the children replay). + ## # Forward the project's configuration to the per-module child processes. # Non-incremental compilation semchecks every module in one process with one # define set (the project's config files apply to the stdlib too); the IC @@ -646,42 +640,61 @@ proc generateBuildFile(c: DepContext): string = # bodies — and thus their import sets and NIF contents — would silently # diverge from the dependency graph computed here. Forward every define that # is not part of the compiler's built-in baseline, plus the threads switch. - var forwardedArgs: seq[string] = @[] - block: - let baseline = newStringTable(modeStyleInsensitive) - initDefines(baseline) - for k, v in pairs(c.config.symbols): - if not baseline.hasKey(k) or baseline[k] != v: - forwardedArgs.add "--define:" & k & (if v == "true": "" else: "=" & v) - sort forwardedArgs - forwardedArgs.add "--threads:" & (if optThreads in c.config.globalOptions: "on" else: "off") - # Forward the memory-management mode too: the children would otherwise - # compile with the default GC while the dependency graph here was computed - # with the selected one (e.g. under --mm:refc the scanner keeps - # system/gc's transitive imports but default-orc children never compile - # them — phantom outputs that re-fire the build on every rerun). - if c.config.selectedGC != gcUnselected: - forwardedArgs.add "--mm:" & $c.config.selectedGC - # method dispatch semantics must match across the child processes: - # a child compiled without --multimethods:on builds different dispatch - # buckets (and rejects calls as ambiguous that multi-dispatch accepts) - if optMultiMethods in c.config.globalOptions: - forwardedArgs.add "--multimethods:on" - # the children compile each MODULE as their own project file, which makes - # that module's package the "main package" and unfilters foreign-package - # diagnostics — a vendored package's hintAsError/warningAsError promotions - # then abort builds the whole-program compilation accepts. Forward the - # real project so children filter diagnostics identically. - forwardedArgs.add "--icproject:" & c.config.projectFull.string - # Precompiled config: serialise the driver's config once and have every - # child replay it instead of re-parsing the `nim.cfg` chain and re-running - # `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig` - # restores the old per-child config parsing (for bisecting a suspected - # config-replay divergence without clearing caches). - if not isDefined(c.config, "icNoPreparsedConfig"): - let cfgArtifact = nimcache / "ic_config.cfg.nif" - writeIcConfig(c.config, cfgArtifact) - forwardedArgs.add "--icPreparsedConfig:" & cfgArtifact + let nimcache = getNimcacheDir(c.config).string + result = @[] + let baseline = newStringTable(modeStyleInsensitive) + initDefines(baseline) + for k, v in pairs(c.config.symbols): + if not baseline.hasKey(k) or baseline[k] != v: + result.add "--define:" & k & (if v == "true": "" else: "=" & v) + sort result + result.add "--threads:" & (if optThreads in c.config.globalOptions: "on" else: "off") + # Forward the memory-management mode too: the children would otherwise + # compile with the default GC while the dependency graph here was computed + # with the selected one (e.g. under --mm:refc the scanner keeps + # system/gc's transitive imports but default-orc children never compile + # them — phantom outputs that re-fire the build on every rerun). + if c.config.selectedGC != gcUnselected: + result.add "--mm:" & $c.config.selectedGC + # method dispatch semantics must match across the child processes: + # a child compiled without --multimethods:on builds different dispatch + # buckets (and rejects calls as ambiguous that multi-dispatch accepts) + if optMultiMethods in c.config.globalOptions: + result.add "--multimethods:on" + # the children compile each MODULE as their own project file, which makes + # that module's package the "main package" and unfilters foreign-package + # diagnostics — a vendored package's hintAsError/warningAsError promotions + # then abort builds the whole-program compilation accepts. Forward the + # real project so children filter diagnostics identically. + result.add "--icproject:" & c.config.projectFull.string + # Precompiled config: serialise the driver's config once and have every + # child replay it instead of re-parsing the `nim.cfg` chain and re-running + # `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig` + # restores the old per-child config parsing (for bisecting a suspected + # config-replay divergence without clearing caches). + if not isDefined(c.config, "icNoPreparsedConfig"): + let cfgArtifact = nimcache / "ic_config.cfg.nif" + writeIcConfig(c.config, cfgArtifact) + result.add "--icPreparsedConfig:" & cfgArtifact + +proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): string = + ## Frontend build file: the nifler (parse) and `nim m` (sem) rules only. The + ## driver runs this to a discovery fixpoint; it produces every module's semmed + ## NIF plus the cookie/edge sidecars that the backend build file then consumes. + ## The backend step lives in its own nifmake run (generateBackendBuildFile) so + ## that "which TUs rebuild" stays a pure nifmake mtime decision rather than + ## something the driver interleaves with the `.s.deps` discovery loop. This + ## split is also the scaffold for the per-module backend: once the backend is + ## per-module, its rules slot into the backend file unchanged. + let nimcache = getNimcacheDir(c.config).string + createDir(nimcache) + result = nimcache / c.nodes[0].files[0].modname & ".frontend.build.nif" + + var b = nifbuilder.open(result) + defer: b.close() + + b.addHeader("nim ic", "nifmake") + b.addTree "stmts" # Define nifler command b.addTree "cmd" @@ -712,22 +725,6 @@ proc generateBuildFile(c: DepContext): string = b.addIntLit 0 # main parsed file b.endTree() - # Define nim nifc command - b.addTree "cmd" - b.addSymbolDef "nim_nifc" - b.addStrLit getAppFilename() - b.addStrLit "nifc" - b.addStrLit "--nimcache:" & nimcache - # Add search paths - for p in c.config.searchPaths: - b.addStrLit "--path:" & p.string - for a in forwardedArgs: - b.addStrLit a - b.addTree "input" - b.addIntLit 0 - b.endTree() - b.endTree() - # Build rules for parsing (nifler) var seenFiles = initHashSet[string]() for node in c.nodes: @@ -867,6 +864,42 @@ proc generateBuildFile(c: DepContext): string = b.endTree() b.endTree() + b.endTree() # stmts + +proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string = + ## Backend build file: today a single whole-program `nim nifc` rule that reads + ## every module's semmed NIF and emits/compiles/links the executable. The + ## driver runs it once, after the frontend fixpoint (the semmed NIFs already + ## exist on disk, so they enter this graph as leaf inputs with no producing + ## rule, exactly like the nifler rules' source `.nim` inputs). The per-module + ## backend rewrite replaces this one rule with per-module codegen + a DCE rule + ## + a link rule; nothing else here changes. + let nimcache = getNimcacheDir(c.config).string + createDir(nimcache) + result = nimcache / c.nodes[0].files[0].modname & ".backend.build.nif" + + var b = nifbuilder.open(result) + defer: b.close() + + b.addHeader("nim ic", "nifmake") + b.addTree "stmts" + + # Define nim nifc command + b.addTree "cmd" + b.addSymbolDef "nim_nifc" + b.addStrLit getAppFilename() + b.addStrLit "nifc" + b.addStrLit "--nimcache:" & nimcache + # Add search paths + for p in c.config.searchPaths: + b.addStrLit "--path:" & p.string + for a in forwardedArgs: + b.addStrLit a + b.addTree "input" + b.addIntLit 0 + b.endTree() + b.endTree() + # Final compilation step: generate executable from main module let mainNif = c.nodes[0].files[0].nimFile let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt) @@ -966,25 +999,35 @@ proc commandIc*(conf: ConfigRef) = # from those sidecars — adding any module the scanner missed, plus the edge # from its importer — and rerun; nifmake's mtime pruning keeps completed # work. A round that discovers nothing new but still fails is a real error. - var rounds = 0 - while true: - let buildFile = generateBuildFile(c) - rawMessage(conf, hintSuccess, "generated: " & buildFile) + let forwardedArgs = computeForwardedArgs(c) + let nifmake = findNifmake() + # Build the per-module rules concurrently: nifmake fans out all commands at + # each DAG depth via execProcesses (defaults to all cores). Cold builds are + # otherwise serial (one child at a time) and leave the machine idle. Opt out + # with `-d:icNoParallel` (e.g. for readable, non-interleaved child output + # when debugging a build). + let parallel = if isDefined(conf, "icNoParallel"): "" else: " --parallel" - let nifmake = findNifmake() - # Build the per-module `nim m` rules concurrently: nifmake fans out all - # commands at each DAG depth via execProcesses (defaults to all cores). - # Cold builds are otherwise serial (one `nim m` at a time) and leave the - # machine idle. Opt out with `-d:icNoParallel` (e.g. for readable, non- - # interleaved child output when debugging a build). - let parallel = if isDefined(conf, "icNoParallel"): "" else: " --parallel" + # Phase 1 — frontend (nifler + `nim m`), run to a discovery fixpoint. + var rounds = 0 + var frontendOk = false + while true: + let buildFile = generateFrontendBuildFile(c, forwardedArgs) + rawMessage(conf, hintSuccess, "generated: " & buildFile) if nifmake.len == 0: rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & buildFile) - break + # without nifmake we can only print the manual commands; emit the + # backend's too (best effort — discovery cannot run) and stop. + let backendFile = generateBackendBuildFile(c, forwardedArgs) + rawMessage(conf, hintSuccess, "generated: " & backendFile) + rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & backendFile) + return let cmd = quoteShell(nifmake) & " run" & parallel & " " & quoteShell(buildFile) rawMessage(conf, hintExecuting, cmd) let exitCode = execShellCmd(cmd) - if exitCode == 0: break + if exitCode == 0: + frontendOk = true + break # Re-derive from the post-sem deps of every node compiled so far. Imports # the static scanner missed become new nodes; the importer->import edge @@ -1014,5 +1057,17 @@ proc commandIc*(conf: ConfigRef) = if not discovered: rawMessage(conf, errGenerated, "nifmake failed with exit code: " & $exitCode) break + + # Phase 2 — backend (whole-program `nim nifc`), run once over the now-final + # graph. Kept a separate nifmake run so backend rebuilds are decided purely + # by nifmake's input mtimes, independent of frontend discovery. + if frontendOk: + let backendFile = generateBackendBuildFile(c, forwardedArgs) + rawMessage(conf, hintSuccess, "generated: " & backendFile) + let cmd = quoteShell(nifmake) & " run" & parallel & " " & quoteShell(backendFile) + rawMessage(conf, hintExecuting, cmd) + let exitCode = execShellCmd(cmd) + if exitCode != 0: + rawMessage(conf, errGenerated, "nifmake (backend) failed with exit code: " & $exitCode) else: rawMessage(conf, errGenerated, "nim ic not available in bootstrap build")