IC: precompiled configs and bugfixes

This commit is contained in:
Araq
2026-06-15 08:16:34 +02:00
parent 7171e6f01f
commit 8938716326
9 changed files with 263 additions and 32 deletions

View File

@@ -508,6 +508,7 @@ proc parseCommand*(command: string): Command =
of "jsonscript": cmdJsonscript
of "nifc": cmdNifC # generate C from NIF files
of "ic": cmdIc # generate .build.nif for nifmake
of "icconfig": cmdIcConfig # produce the precompiled config artifact
else: cmdUnknown
proc setCmd*(conf: ConfigRef, cmd: Command) =
@@ -960,6 +961,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
# config loading can replay it instead of re-parsing the `nim.cfg` chain.
expectArg(conf, switch, arg, pass, info)
conf.icPreparsedConfig = arg
of "icconfigout":
# `nim icconfig` only: where to write the precompiled config artifact (see
# options.icConfigOut). The `nim ic` driver spawns the producer with this.
expectArg(conf, switch, arg, pass, info)
conf.icConfigOut = arg
of "icbackendstage":
# `nim nifc` only: per-module backend stage, one of cg|merge|emit (see
# options.icBackendStage). Empty (switch unused) keeps the whole-program

View File

@@ -11,7 +11,7 @@
## This enables incremental and parallel compilation using the `m` switch.
import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio]
import options, msgs, lineinfos, pathutils, condsyms, icconfig,
import options, msgs, lineinfos, pathutils, condsyms,
modulepaths, extccomp, cnif
import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder]
@@ -211,6 +211,15 @@ proc processInclude(c: var DepContext; includePath: string; current: Node) =
traverseDeps(c, c.toPair(resolved), current)
discard c.includeStack.pop()
proc getsImplicitImports(c: DepContext; nimFile: string): bool =
## Mirror the compiler's `belongsToStdlib` guard (pipelines.nim): `--import:X`
## (conf.implicitImports) is applied only to NON-stdlib modules. The scanner
## must agree, otherwise it edges a stdlib module → X that the compiler never
## actually creates, fabricating a cycle that folds X — and the modules X
## claims to produce — into the system SCC (whose `nim m` is driven from
## system.nim and never reaches them). Stdlib == under conf.libpath.
not isRelativeTo(nimFile, c.config.libpath.string)
proc processImport(c: var DepContext; importPath: string; current: Node) =
let resolved = resolveImport(c, current.files[0].nimFile, importPath)
if resolved.len == 0 or not fileExists(resolved):
@@ -226,12 +235,15 @@ proc processImport(c: var DepContext; importPath: string; current: Node) =
# Every module depends on system.nim
if c.systemNodeId >= 0:
newNode.deps.add c.systemNodeId
# ... and on every `--import`ed module (conf.implicitImports). A `--import`ed
# module is itself imported by its own closure (which also gets these edges),
# so the cycle folds into one strongly-connected component (see computeSCCs),
# just like system.nim's closure.
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
# ... and on every `--import`ed module (conf.implicitImports), but only for
# the non-stdlib modules the compiler actually applies implicit imports to
# (see getsImplicitImports). A `--import`ed module is imported by its own
# non-stdlib closure (which also gets these edges), so that cycle folds into
# one small strongly-connected component (see computeSCCs) instead of being
# smeared across system + stdlib.
if getsImplicitImports(c, pair.nimFile):
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
c.processedModules[pair.modname] = newNode.id
c.nodes.add newNode
traverseDeps(c, pair, newNode)
@@ -694,15 +706,14 @@ proc computeForwardedArgs(c: DepContext): seq[string] =
# then abort builds the whole-program compilation accepts. Forward the
# real project so children filter diagnostics identically.
result.add "--icproject:" & c.config.projectFull.string
# Precompiled config: serialise the driver's config once and have every
# child replay it instead of re-parsing the `nim.cfg` chain and re-running
# `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig`
# restores the old per-child config parsing (for bisecting a suspected
# config-replay divergence without clearing caches).
if not isDefined(c.config, "icNoPreparsedConfig"):
let cfgArtifact = nimcache / "ic_config.cfg.nif"
writeIcConfig(c.config, cfgArtifact)
result.add "--icPreparsedConfig:" & cfgArtifact
# Precompiled config: every child replays the one artifact produced (in a
# separate `nim icconfig` process) and already replayed by the driver itself —
# see `icconfig.ensureIcConfig`, run before the driver's own `loadConfigs`. So
# `nim ic` is always governed by this single artifact, for speed and so the
# driver and its children agree by construction. Forward the path the driver
# replayed (`conf.icPreparsedConfig`); `commandIc` has already guaranteed it
# exists, else it bailed.
result.add "--icPreparsedConfig:" & c.config.icPreparsedConfig
proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
## Frontend build file: the nifler (parse) and `nim m` (sem) rules only. The
@@ -1029,6 +1040,12 @@ proc commandIc*(conf: ConfigRef) =
rawMessage(conf, errGenerated, "nifler tool not found. Install nimony or add nifler to PATH.")
return
# Resolve the `.nim` source first, exactly like `wantMainModule`. Without
# this, an extensionless project arg (`nim ic path/to/foo`) resolves to a
# same-named sibling that already exists — e.g. the ELF a prior `nim c`
# left behind — and nifler chokes on the binary (`invalid token \127`,
# ELF magic). `addFileExt` only appends when there is no extension.
conf.projectFull = addFileExt(conf.projectFull, NimExt)
let projectFile = conf.projectFull.string
if not fileExists(projectFile):
rawMessage(conf, errGenerated, "project file not found: " & projectFile)
@@ -1118,6 +1135,15 @@ proc commandIc*(conf: ConfigRef) =
# from its importer — and rerun; nifmake's mtime pruning keeps completed
# work. A round that discovers nothing new but still fails is a real error.
let forwardedArgs = computeForwardedArgs(c)
# The precompiled config drives every `nim m`/`nim nifc` child and the driver
# itself (`ensureIcConfig` produced it and `loadConfigs` replayed it). If it
# is not on disk something went wrong producing it — children would each
# silently fall back to re-parsing the whole config chain — so refuse to
# continue without it.
if conf.icPreparsedConfig.len == 0 or not fileExists(conf.icPreparsedConfig):
rawMessage(conf, errGenerated,
"precompiled config missing: " & conf.icPreparsedConfig)
return
let nifmake = findNifmake()
# Build the per-module rules concurrently: nifmake fans out all commands at
# each DAG depth via execProcesses (defaults to all cores). Cold builds are
@@ -1164,8 +1190,9 @@ proc commandIc*(conf: ConfigRef) =
let newNode = Node(files: @[pair], id: c.nodes.len)
if c.systemNodeId >= 0:
newNode.deps.add c.systemNodeId
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
if getsImplicitImports(c, pair.nimFile):
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
c.processedModules[pair.modname] = newNode.id
c.nodes.add newNode
idx = newNode.id

View File

@@ -31,23 +31,39 @@
## `--path` arguments; replaying their raw, config-dir-relative arguments here
## would misresolve.
import options, commands, lineinfos
import std/[algorithm, os, sets]
import options, commands, lineinfos, pathutils, msgs
import std/[algorithm, os, sets, osproc, times, streams, syncio]
import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse]
const
IcConfigVersion* = "1"
IcConfigVersion* = "2"
## Artifact format version. Bump on any layout change here so a child built
## by an older compiler rejects a stale artifact and falls back to normal
## config loading instead of replaying a format it cannot parse.
proc writeIcConfig*(conf: ConfigRef; outfile: string) =
## Serialise the config-file switches recorded during `loadConfigs` plus the
## resolved `cppDefines` set into the artifact at `outfile`.
var b = nifbuilder.open(outfile)
## Serialise the resolved config (the config-file switches recorded during
## `loadConfigs`, the resolved `cppDefines`/`searchPaths`, the nimcache dir, and
## the list of config *source* files for staleness detection) into `outfile`.
## `OnlyIfChanged`: when the content is byte-identical to what is already on
## disk the file is left untouched so its mtime does not advance — otherwise
## every `nim ic` run would re-fire the whole nifmake graph (see `nifler`'s
## `produceConfig`, whose model this mirrors).
var b = nifbuilder.open(outfile, writeMode = OnlyIfChanged)
b.withTree "stmts":
b.withTree "meta":
b.addStrLit IcConfigVersion
b.withTree "sources":
# Every config file read while loading (nim.cfg chain + config.nims), so a
# later run can decide via mtimes whether this artifact is still current
# (see `sourcesChanged`).
for f in conf.configFiles:
b.addStrLit f.string
b.withTree "nimcache":
# Resolved build nimcache. Recorded (unlike the path-search switches) so the
# driver, which replays this artifact instead of parsing `nim.cfg`, still
# learns a `--nimcache:` set inside `nim.cfg` and builds in the right place.
b.addStrLit conf.nimcacheDir.string
b.withTree "cppdefines":
# HashSet iteration order is unspecified; sort so the artifact is
# byte-stable across runs (nifmake keys rebuilds off content changes).
@@ -55,6 +71,15 @@ proc writeIcConfig*(conf: ConfigRef; outfile: string) =
for d in conf.cppDefines: defs.add d
sort defs
for d in defs: b.addStrLit d
b.withTree "searchpaths":
# The resolved (absolute) search paths. Path-search *switches* are skipped
# below because their raw arguments are config-dir-relative; the net effect
# lives here instead, so a replayer with no `--path` command-line arguments
# (the `nim ic` driver itself) still resolves imports. `nim m`/`nim nifc`
# children also receive these as forwarded `--path` args; the dedup on
# replay makes the overlap harmless.
for p in conf.searchPaths:
b.addStrLit p.string
b.withTree "switches":
for sw in conf.icConfigSwitches:
b.addTree "sw"
@@ -74,7 +99,10 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
let
stmtsTag = tags.registerTag("stmts")
metaTag = tags.registerTag("meta")
sourcesTag = tags.registerTag("sources")
nimcacheTag = tags.registerTag("nimcache")
cppTag = tags.registerTag("cppdefines")
pathsTag = tags.registerTag("searchpaths")
switchesTag = tags.registerTag("switches")
swTag = tags.registerTag("sw")
var buf = parseFromFile(infile, 1000, pool, tags)
@@ -95,6 +123,22 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
inc c
else:
skip c
elif c.cursorTagId == nimcacheTag:
c.loopInto:
if c.kind == StrLit:
let nc = strVal(c)
# Only when nimcache was not already pinned on the command line: a
# `--nimcache:` argument the driver/child was launched with must win
# over whatever `nim.cfg` recorded into the artifact.
if nc.len > 0 and conf.nimcacheDir.isEmpty:
conf.nimcacheDir = AbsoluteDir(nc)
inc c
else:
skip c
elif c.cursorTagId == sourcesTag:
# Replay does not need the source list; it exists only for
# `sourcesChanged`. Skip the whole section.
skip c
elif c.cursorTagId == cppTag:
c.loopInto:
if c.kind == StrLit:
@@ -102,6 +146,17 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
inc c
else:
skip c
elif c.cursorTagId == pathsTag:
c.loopInto:
if c.kind == StrLit:
# Append preserving the serialised order (which already reflects the
# driver's addPath insert-at-front sequence), deduping against any
# path a child already received via a forwarded `--path` argument.
let d = AbsoluteDir(strVal(c))
if not conf.searchPaths.contains(d): conf.searchPaths.add d
inc c
else:
skip c
elif c.cursorTagId == switchesTag:
c.loopInto:
if c.kind == TagLit and c.cursorTagId == swTag:
@@ -125,3 +180,104 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
skip c
endRead(c)
result = sawMeta and version == IcConfigVersion
proc sourcesChanged*(configFile: string): bool =
## True when the precompiled config at `configFile` is missing, malformed,
## written by an incompatible version, or any recorded config *source* file is
## newer than it (or has vanished) — i.e. the artifact must be regenerated.
## Mirrors nifler's `sourcesChanged`: the source list lives inside the artifact
## so this needs no out-of-band knowledge of which `nim.cfg`s were read.
if not fileExists(configFile): return true
let modtime = getLastModificationTime(configFile)
var pool = newPool()
var tags = newTagPool()
let
stmtsTag = tags.registerTag("stmts")
metaTag = tags.registerTag("meta")
sourcesTag = tags.registerTag("sources")
var buf = parseFromFile(configFile, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return true
var version = ""
var depsChanged = false
c.loopInto:
if c.kind == TagLit and c.cursorTagId == metaTag:
c.loopInto:
if c.kind == StrLit:
version = strVal(c)
inc c
else:
skip c
elif c.kind == TagLit and c.cursorTagId == sourcesTag:
c.loopInto:
if c.kind == StrLit:
let dep = strVal(c)
if not fileExists(dep) or getLastModificationTime(dep) >= modtime:
depsChanged = true
inc c
else:
skip c
else:
skip c
endRead(c)
result = depsChanged or version != IcConfigVersion
proc produceIcConfig*(conf: ConfigRef) =
## The `cmdIcConfig` command. By the time it runs, the normal pipeline has
## already fully parsed the `nim.cfg` chain and run `config.nims`, so the
## resolved config is sitting in `conf`; just serialise it to `--o`.
let outPath = conf.icConfigOut
if outPath.len == 0:
rawMessage(conf, errGenerated, "icconfig: missing output path (--icConfigOut)")
return
createDir(parentDir(outPath))
writeIcConfig(conf, outPath)
proc ensureIcConfig*(conf: ConfigRef) =
## Driver-side (`cmdIc`). Make sure an up-to-date precompiled config exists,
## (re)producing it in a *separate* process when missing or stale, then point
## `conf.icPreparsedConfig` at it so the driver replays the very same config its
## `nim m`/`nim nifc` children will — perfect speed (config parsed at most once,
## skipped entirely when nothing changed) and consistency (one producer, every
## process replays its output). The artifact lives in the nimcache derived from
## the command line (pre-config-parse), which is the one the children are told;
## a `--nimcache:` set inside `nim.cfg` is recovered from the artifact itself.
let cacheDir = getNimcacheDir(conf).string
# Start from a clean cache when the on-disk NIF format stamp is absent or stale
# (see `icFormatVersion`). This must happen HERE, before the config artifact is
# produced — `commandIc` performs the same check later, but by then the artifact
# would already live in the cache and the wipe would delete it.
createDir(cacheDir)
let versionFile = cacheDir / "ic.version"
let stamp = if fileExists(versionFile): readFile(versionFile) else: ""
if stamp != icFormatVersion:
removeDir(cacheDir)
createDir(cacheDir)
writeFile(versionFile, icFormatVersion)
let outPath = cacheDir / "ic_config.cfg.nif"
if not fileExists(outPath) or sourcesChanged(outPath):
createDir(cacheDir)
# Re-invoke ourselves as the config producer: reuse this process's command
# line, dropping the command argument (`ic`) in favour of `icconfig` and the
# explicit output path, both BEFORE the project file (anything after the
# project is swallowed into `config.arguments` by `cmdLineRest`). The
# producer re-reads `nim.cfg` itself.
var pargs = @["icconfig", "--icConfigOut:" & outPath]
var droppedCmd = false
for a in commandLineParams():
if not droppedCmd and a.len > 0 and a[0] != '-':
droppedCmd = true # drop the original command token (`ic`)
else:
pargs.add a
let p = startProcess(getAppFilename(), args = pargs,
options = {poStdErrToStdOut})
let outp = p.outputStream.readAll()
let code = p.waitForExit()
p.close()
if code != 0 or not fileExists(outPath):
rawMessage(conf, errGenerated,
"failed to produce precompiled config (exit code " & $code & "):\n" & outp)
return
conf.icPreparsedConfig = outPath

View File

@@ -29,6 +29,7 @@ when defined(nimPreviewSlimSystem):
import ../dist/checksums/src/checksums/sha1
import pipelines
from icconfig import produceIcConfig
when not defined(nimKochBootstrap):
import nifbackend
@@ -439,6 +440,11 @@ proc mainCommand*(graph: ModuleGraph) =
commandIc(conf)
else:
rawMessage(conf, errGenerated, "nim deps not available in bootstrap build")
of cmdIcConfig:
# Produce the precompiled config artifact for `nim ic` (config already
# parsed by the normal pipeline); a separate process spawned by the driver.
wantMainModule(conf)
produceIcConfig(conf)
of cmdParse:
wantMainModule(conf)
discard parseFile(conf.projectMainIdx, cache, conf)

View File

@@ -1063,7 +1063,16 @@ proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
proc belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool =
## Check if symbol belongs to the 'stdlib' package.
sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId
# Compare the package *name* (an interned ident), not the package symbol's
# `.id`. Under per-module IC (`nim m`) the system module is loaded from a NIF
# in a process that does not compile it from source, so its package symbol is
# reconstructed with a fresh `.id` that no longer matches the freshly-interned
# package of a stdlib module compiled standalone here — making the old id
# comparison wrongly report `false` and inject `--import`ed modules into the
# stdlib. Both are canonically named `stdlib` (lib/stdlib.nimble); in a normal
# `nim c` build (system compiled from source) the ids match too, so this is a
# no-op there.
sym.getPackageSymbol.name.id == graph.systemModule.getPackageSymbol.name.id
proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): SuggestFileSymbolDatabase =
result = graph.suggestSymbols.getOrDefault(fileIdx, newSuggestFileSymbolDatabase(fileIdx, optIdeExceptionInlayHints in graph.config.globalOptions))

View File

@@ -29,6 +29,7 @@ import
pathutils, modulegraphs
from ast2nif import registerNifAstTags
from icconfig import ensureIcConfig
from std/browsers import openDefaultBrowser
from nodejs import findNodeJs
@@ -114,6 +115,14 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
self.processCmdLineAndProjectPath(conf)
# `nim ic` driver: ensure the precompiled config exists (produced by a separate
# `nim icconfig` process, skipped when nothing changed) BEFORE config loading,
# so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the
# driver runs on the exact same config its children will. See icconfig.nim.
when not defined(nimKochBootstrap):
if conf.cmd == cmdIc:
ensureIcConfig(conf)
var graph = newModuleGraph(cache, conf)
if not self.loadConfigsAndProcessCmdLine(cache, conf, graph):
return

View File

@@ -246,10 +246,14 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile
proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) =
setDefaultLibpath(conf)
# `nim ic` children replay the precompiled config the driver recorded once,
# instead of re-reading the `nim.cfg` chain and re-running `config.nims` in the
# VM. A missing/format-incompatible artifact returns false: fall through to
# normal config loading so an older child or a deleted cache still works.
# The `nim ic` driver and its `nim m`/`nim nifc` children replay the precompiled
# config (produced once by a separate `nim icconfig` process — see
# `icconfig.ensureIcConfig`, which sets `icPreparsedConfig` for the driver
# before this runs; the children get it as a forwarded `--icPreparsedConfig`
# argument) instead of re-reading the `nim.cfg` chain and re-running
# `config.nims` in the VM. A missing/format-incompatible artifact returns false:
# fall through to a normal parse (this is also the path the `nim icconfig`
# producer itself takes, since it runs with no `icPreparsedConfig`).
if conf.icPreparsedConfig.len > 0 and applyIcConfig(conf, conf.icPreparsedConfig):
return
template readConfigFile(path) =

View File

@@ -200,6 +200,7 @@ type
cmdCompileToNif
cmdNifC # generate C code from NIF files
cmdIc # generate .build.nif for nifmake
cmdIcConfig # `nim ic`'s precompiled-config producer (writes ic_config.cfg.nif)
const
cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,
@@ -418,12 +419,16 @@ type
# module's package the "main package" and unfilter
# foreign-package diagnostics; the real project
# restores whole-program filtering semantics.
icPreparsedConfig*: string # under `nim m`/`nim nifc`: path of the precompiled
# config artifact written once by the `nim ic` driver.
icPreparsedConfig*: string # under the `nim ic` driver and its `nim m`/`nim nifc`
# children: path of the precompiled config artifact.
# When set, `loadConfigs` replays the recorded
# config-file switches from it instead of re-reading
# the `nim.cfg` chain and re-running `config.nims`
# (which the VM makes expensive) per subprocess.
# (which the VM makes expensive) per process. The
# artifact itself is produced by a separate
# `nim icconfig` process (see `cmdIcConfig`).
icConfigOut*: string # under `nim icconfig`: the path to write the
# precompiled config artifact to (set via `--o`).
icConfigSwitches*: seq[tuple[switch, arg: string]]
# the config-file (`passPP`) switches applied while
# loading config, in order. Recorded by every nim

View File

@@ -286,6 +286,15 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
result = graph.getModule(fileIdx)
template processModuleAux(moduleStatus) =
when defined(icDbg):
block:
let dbgf = open("/tmp/defdbg.txt", fmAppend)
dbgf.writeLine toFullPath(graph.config, fileIdx) &
" nimStackTraceOverride=" & $isDefined(graph.config, "nimStackTraceOverride") &
" nimscript=" & $isDefined(graph.config, "nimscript") &
" optCompress=" & $(optCompress in graph.config.globalOptions) &
" cmd=" & $graph.config.cmd
dbgf.close()
onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
var s: PLLStream = nil
if sfMainModule in flags: