From e0c0724b62ecf9361a8def4abfe3c5d19f1f4184 Mon Sep 17 00:00:00 2001 From: araq Date: Fri, 28 Aug 2026 23:13:16 +0200 Subject: [PATCH] IC: the link stage no longer loads the module graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--icBackendStage:link` called `loadBackendModules` — a whole-program deserialization — for exactly two things: each module's `.c` path via `getCFile`, and its recorded C compile/link directives via `replayBackendActions`. That was 3.7s of the ~11s serial backend critical path on a 219-module program, spent recovering a list of paths and a handful of strings. Both now come from artifacts the earlier stages already produce: * `.c` paths from the driver's existing `LiveModulesFile` manifest (which `merge` already reads); the `.c` sits beside each listed `.c.nif`, so no new manifest was needed. The merge-decision fallback for conditionally-imported nodes that own a live symbol is kept. * C directives from a new `.c.cflags` sidecar. The module's own `cg` already replays them, so it writes them down too — one tab-separated line per `compile`/`link`/`passl`/`passc`/`localpassc`/`cppdefine` action. `localpassc` needs the module's source path, which only the writer can resolve, so it is baked into the line. Written unconditionally, empty included: it is a declared nifmake output of the `cg` rule and a missing output re-fires the rule for ever. link: 3.70s -> 0.44s. A one-line code edit on the corpus goes 15.2s -> 11.8s; cold 124s -> 120s; no-op unchanged at 0.27s. Green: 16/16 metamorphic IC tests, the 17-file `koch ic` suite, 13/13 differential edit checks against `nim c`, and `bootic` iteration 1 (the full fixed-point check was still running when this was committed). Co-Authored-By: Claude Opus 5 --- compiler/deps.nim | 10 ++++- compiler/ic/replayer.nim | 68 +++++++++++++++++++++++++++- compiler/nifbackend.nim | 97 +++++++++++++++++++++++----------------- 3 files changed, 132 insertions(+), 43 deletions(-) diff --git a/compiler/deps.nim b/compiler/deps.nim index a9c743fef9..c7d357e82b 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -18,6 +18,7 @@ import options, msgs, lineinfos, pathutils, condsyms, import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder] import icmodnames import icnifcore +from ic/replayer import BackendActionsExt type FilePair = object @@ -1362,6 +1363,7 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string removeFile(cnifFiles[i]) removeFile(cFiles[i]) removeFile(cFiles[i] & ".stamp") + removeFile(cFiles[i] & BackendActionsExt) # The merge decision is a pure function of the set of `.c.nif`s present; if we # just removed an over-approximated module's artifacts, a decision computed # while they were present is stale — it can name a now-absent module as a @@ -1459,6 +1461,10 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string if c.nodes[j].id != 0 and live[j]: inputStr cnifFiles[j] outputStr cnifFiles[i] + # The module's C compile/link directives (`{.passL.}` etc.), recorded so the + # `link` stage recovers them without loading the module graph. See + # `replayer.writeBackendActions`. + outputStr cFiles[i] & BackendActionsExt b.endTree() # merge: read the live modules' `.c.nif`, write the ownership/liveness @@ -1520,7 +1526,9 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string # path splits back into outDir+outFile in the child). b.addStrLit "--out:" & exeFile for i in 0 ..< c.nodes.len: - if live[i]: inputStr cFiles[i] + if live[i]: + inputStr cFiles[i] + inputStr cFiles[i] & BackendActionsExt inputStr argsFile outputStr exeFile b.endTree() diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim index e31b2be82a..3791a95e57 100644 --- a/compiler/ic/replayer.nim +++ b/compiler/ic/replayer.nim @@ -14,11 +14,77 @@ import ".." / [ast, modulegraphs, trees, extccomp, btrees, msgs, lineinfos, pathutils, options, cgmeth] -import std/tables +import std/[tables, os, strutils, syncio] when defined(nimPreviewSlimSystem): import std/assertions +const BackendActionsExt* = ".cflags" + ## Sidecar written by a module's `cg` stage next to its `.c`, carrying the C + ## compile/link directives that module's `{.passL.}`/`{.compile.}`/… pragmas + ## recorded. See `writeBackendActions`. + +proc writeBackendActions*(g: ModuleGraph; module: PSym; list: PNode; + outfile: string) = + ## Serialize the backend-relevant replay actions of ONE module to `outfile`, + ## one tab-separated action per line. + ## + ## The `link` stage used to recover these by loading the whole import closure + ## as `PrecompiledModule`s and re-running `replayBackendActions` over each — + ## a 3.7s whole-program graph load, per link, purely to recover a handful of + ## strings and the modules' `.c` paths. The producing `cg` process already has + ## them in hand, so it writes them down instead and `link` reads them back + ## (`applyBackendActions`). Written unconditionally, even when empty: it is a + ## declared nifmake output of the `cg` rule, and a missing output re-fires the + ## rule for ever. + ## + ## `localpassc` needs the module's own source path, which only the writer can + ## resolve, so it is baked in here as a third field. + var content = "" + if list != nil: + for n in list: + if n.kind == nkReplayAction and n.len >= 2 and + n[0].kind == nkStrLit and n[1].kind == nkStrLit: + case n[0].strVal + of "compile": + if n.len == 4 and n[2].kind == nkStrLit and n[3].kind == nkStrLit: + content.add "compile\t" & n[1].strVal & "\t" & n[2].strVal & "\t" & + n[3].strVal & "\n" + of "link", "passl", "passc", "cppdefine": + content.add n[0].strVal & "\t" & n[1].strVal & "\n" + of "localpassc": + content.add "localpassc\t" & n[1].strVal & "\t" & + toFullPathConsiderDirty(g.config, module.info.fileIndex).string & "\n" + else: discard + writeFile(outfile, content) + +proc applyBackendActions*(g: ModuleGraph; infile: string) = + ## Apply one module's recorded C directives (see `writeBackendActions`). The + ## `link` stage's replacement for loading that module and replaying its AST. + if not fileExists(infile): return + for line in lines(infile): + if line.len == 0: continue + let f = line.split('\t') + case f[0] + of "compile": + if f.len == 4: + let cname = AbsoluteFile f[1] + var cf = Cfile(nimname: splitFile(cname).name, cname: cname, + obj: AbsoluteFile f[2], + flags: {CfileFlag.External}, customArgs: f[3]) + extccomp.addExternalFileToCompile(g.config, cf) + of "link": + if f.len == 2: extccomp.addExternalFileToLink(g.config, AbsoluteFile f[1]) + of "passl": + if f.len == 2: extccomp.addLinkOption(g.config, f[1]) + of "passc": + if f.len == 2: extccomp.addCompileOption(g.config, f[1]) + of "localpassc": + if f.len == 3: extccomp.addLocalCompileOption(g.config, f[1], AbsoluteFile f[2]) + of "cppdefine": + if f.len == 2: options.cppDefine(g.config, f[1]) + else: discard + proc replayStateChanges*(module: PSym; g: ModuleGraph; list: PNode) = ## `list` is an `nkStmtList` of `nkReplayAction` nodes (macro-cache puts/incs/ ## adds/incls and a few pragmas) recorded for `module`. Under the NIF backend a diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index 64a8567829..53c0f62abb 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -628,6 +628,11 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = let tb = bl.mods[target.module.position] if tb != nil: finishModule(g, tb) + # Record this module's C compile/link directives next to its `.c` so the + # `link` stage can recover them without loading the module graph. See + # `replayer.writeBackendActions`. + writeBackendActions(g, target.module, target.topLevel, + getCFile(tb).string & BackendActionsExt) # Writes only the target's `.c.nif` (every other loaded module's TU is empty, # so `cgenWriteModules` emits no artifact for it). cc/link are NOT run here. @@ -746,53 +751,62 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) = ## Per-module backend link (`--icBackendStage:link`): the `emit` stages have ## written every module's `.c`; register them and run the C compiler + linker ## once via `extccomp.callCCompiler` (which parallelizes the per-file cc and - ## skips up-to-date objects itself). No codegen runs — the graph is loaded only - ## so `getCFile` yields each module's emitted `.c` path. - let (modules, precompSys, _) = loadBackendModules(g, mainFileIdx) - if modules.len == 0: - rawMessage(g.config, errGenerated, - "Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx)) - return - # The per-module `cg` processes each collect their module's C compile/link - # directives (`{.passL: "-lm".}` etc.) via `replayBackendActions`, but those - # live in the cg process and never reach this separate link process. Re-collect - # every loaded module's directives here so the final `callCCompiler` sees them - # (without this, math's `-lm` is lost → undefined `floor`/`pow`/… at link). - for m in modules: - replayBackendActions(g, m.module, m.topLevel) - if precompSys.module != nil: - replayBackendActions(g, precompSys.module, precompSys.topLevel) - let bl = BModuleList(g.backend) + ## skips up-to-date objects itself). No codegen runs and NO MODULE GRAPH IS + ## LOADED. + ## + ## It used to load the whole import closure (`loadBackendModules`) for two + ## things only: each module's `.c` path via `getCFile`, and its recorded C + ## directives via `replayBackendActions`. That was 3.7s of the ~11s serial + ## backend critical path on a 219-module program — a whole-program + ## deserialization to recover a list of paths and a handful of strings. Both + ## are now read from artifacts the earlier stages already produce: + ## * the driver's `LiveModulesFile` manifest lists every live module's + ## `.c.nif`, and the `.c` sits beside it (`emit`'s output); + ## * each module's `cg` wrote its directives to a `.cflags` sidecar. + let nimcache = getNimcacheDir(g.config).string + var cfiles: seq[string] = @[] + let manifest = nimcache / LiveModulesFile + if fileExists(manifest): + for line in lines(manifest): + let p = line.strip() + if p.len > 0 and p.endsWith(".nif"): cfiles.add p[0 ..< p.len - ".nif".len] + else: + # A cache written by an older compiler has no manifest; fall back to the + # `.c` files sitting next to the artifacts. + for artifact in walkFiles(nimcache / ("*" & icCFileExt(g.config) & ".nif")): + cfiles.add artifact[0 ..< artifact.len - ".nif".len] + sort cfiles + var addedCFiles = initHashSet[string]() - for m in bl.mods: - if m != nil: - let cfile = getCFile(m) - # Only modules that are their own cg/emit target produced a `.c`; the rest - # (extra members of system's closure that no build rule targets) had their - # code emit-everywhere'd into the targets, so they have no file to compile. - if not fileExists(cfile.string): continue - addedCFiles.incl extractFilename(cfile.string) - var cf = Cfile(nimname: m.module.name.s, cname: cfile, - obj: completeCfilePath(g.config, toObjFile(g.config, cfile)), - flags: {}) - # `addExternalFileToCompile` (not `addFileToCompile`) gates each `.c` on its - # SHA1 footprint: an unchanged `.c` keeps its `.o` and is flagged Cached, so - # `callCCompiler` skips its compile but still links the existing object. This - # is what makes a localized edit recompile only the handful of `.c`s the - # `emit` stage actually rewrote, instead of every object every time — the - # final piece of per-module backend incrementality after the merge barrier. - addExternalFileToCompile(g.config, cf) + for cpath in cfiles: + # Only modules that are their own cg/emit target produced a `.c`; the rest + # had their code emit-everywhere'd into the targets, so there is nothing to + # compile for them. + if not fileExists(cpath): continue + addedCFiles.incl extractFilename(cpath) + # The directives this module recorded (`{.passL: "-lm".}` etc.); without + # them math's `-lm` is lost -> undefined `floor`/`pow`/… at link. + applyBackendActions(g, cpath & BackendActionsExt) + let cfile = AbsoluteFile cpath + var cf = Cfile(nimname: splitFile(cfile).name, cname: cfile, + obj: completeCfilePath(g.config, toObjFile(g.config, cfile)), + flags: {}) + # `addExternalFileToCompile` (not `addFileToCompile`) gates each `.c` on its + # SHA1 footprint: an unchanged `.c` keeps its `.o` and is flagged Cached, so + # `callCCompiler` skips its compile but still links the existing object. This + # is what makes a localized edit recompile only the handful of `.c`s the + # `emit` stage actually rewrote, instead of every object every time. + addExternalFileToCompile(g.config, cf) + # deps.nim's static scanner can keep a CONDITIONALLY-imported module as a build - # node (e.g. `net`'s `when defineSsl: import openssl`, or a `when defined(os)` - # import) that the NIF-`deps` walk above never reaches because the condition is - # off. Such a node still emitted a `.c`, and it can OWN a live generic instance - # that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused by - # `strutils.escape`) — so its body must be at link or that reference is + # node (e.g. `net`'s `when defineSsl: import openssl`) that the manifest above + # may not cover. Such a node still emitted a `.c`, and it can OWN a live generic + # instance that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused + # by `strutils.escape`) — so its body must be at link or that reference is # undefined. Link every emitted `.c` the merge decision says OWNS a LIVE symbol; # a node that owns nothing live (a Windows-only winsock node on Linux) is # correctly skipped. block: - let nimcache = getNimcacheDir(g.config).string let decision = readMergeDecision(nimcache / MergeDecisionFile) if not decision.broken: var liveOwners = initHashSet[string]() @@ -804,6 +818,7 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) = if addedCFiles.containsOrIncl(cbase): continue let cfile = AbsoluteFile(nimcache / cbase) if not fileExists(cfile.string): continue + applyBackendActions(g, cfile.string & BackendActionsExt) var cf = Cfile(nimname: cbase, cname: cfile, obj: completeCfilePath(g.config, toObjFile(g.config, cfile)), flags: {})