diff --git a/compiler/commands.nim b/compiler/commands.nim index f5434fb621..31a185d838 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -508,6 +508,7 @@ proc parseCommand*(command: string): Command = of "jsonscript": cmdJsonscript of "nifc": cmdNifC # generate C from NIF files of "ic": cmdIc # generate .build.nif for nifmake + of "icconfig": cmdIcConfig # produce the precompiled config artifact else: cmdUnknown proc setCmd*(conf: ConfigRef, cmd: Command) = @@ -960,6 +961,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; # config loading can replay it instead of re-parsing the `nim.cfg` chain. expectArg(conf, switch, arg, pass, info) conf.icPreparsedConfig = arg + of "icconfigout": + # `nim icconfig` only: where to write the precompiled config artifact (see + # options.icConfigOut). The `nim ic` driver spawns the producer with this. + expectArg(conf, switch, arg, pass, info) + conf.icConfigOut = arg of "icbackendstage": # `nim nifc` only: per-module backend stage, one of cg|merge|emit (see # options.icBackendStage). Empty (switch unused) keeps the whole-program diff --git a/compiler/deps.nim b/compiler/deps.nim index 34d06ac0e1..7015db6f1a 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -11,7 +11,7 @@ ## This enables incremental and parallel compilation using the `m` switch. import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio] -import options, msgs, lineinfos, pathutils, condsyms, icconfig, +import options, msgs, lineinfos, pathutils, condsyms, modulepaths, extccomp, cnif import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder] @@ -211,6 +211,15 @@ proc processInclude(c: var DepContext; includePath: string; current: Node) = traverseDeps(c, c.toPair(resolved), current) discard c.includeStack.pop() +proc getsImplicitImports(c: DepContext; nimFile: string): bool = + ## Mirror the compiler's `belongsToStdlib` guard (pipelines.nim): `--import:X` + ## (conf.implicitImports) is applied only to NON-stdlib modules. The scanner + ## must agree, otherwise it edges a stdlib module → X that the compiler never + ## actually creates, fabricating a cycle that folds X — and the modules X + ## claims to produce — into the system SCC (whose `nim m` is driven from + ## system.nim and never reaches them). Stdlib == under conf.libpath. + not isRelativeTo(nimFile, c.config.libpath.string) + proc processImport(c: var DepContext; importPath: string; current: Node) = let resolved = resolveImport(c, current.files[0].nimFile, importPath) if resolved.len == 0 or not fileExists(resolved): @@ -226,12 +235,15 @@ proc processImport(c: var DepContext; importPath: string; current: Node) = # Every module depends on system.nim if c.systemNodeId >= 0: newNode.deps.add c.systemNodeId - # ... and on every `--import`ed module (conf.implicitImports). A `--import`ed - # module is itself imported by its own closure (which also gets these edges), - # so the cycle folds into one strongly-connected component (see computeSCCs), - # just like system.nim's closure. - for impId in c.implicitNodeIds: - if impId != newNode.id: newNode.deps.add impId + # ... and on every `--import`ed module (conf.implicitImports), but only for + # the non-stdlib modules the compiler actually applies implicit imports to + # (see getsImplicitImports). A `--import`ed module is imported by its own + # non-stdlib closure (which also gets these edges), so that cycle folds into + # one small strongly-connected component (see computeSCCs) instead of being + # smeared across system + stdlib. + if getsImplicitImports(c, pair.nimFile): + for impId in c.implicitNodeIds: + if impId != newNode.id: newNode.deps.add impId c.processedModules[pair.modname] = newNode.id c.nodes.add newNode traverseDeps(c, pair, newNode) @@ -694,15 +706,14 @@ proc computeForwardedArgs(c: DepContext): seq[string] = # then abort builds the whole-program compilation accepts. Forward the # real project so children filter diagnostics identically. result.add "--icproject:" & c.config.projectFull.string - # Precompiled config: serialise the driver's config once and have every - # child replay it instead of re-parsing the `nim.cfg` chain and re-running - # `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig` - # restores the old per-child config parsing (for bisecting a suspected - # config-replay divergence without clearing caches). - if not isDefined(c.config, "icNoPreparsedConfig"): - let cfgArtifact = nimcache / "ic_config.cfg.nif" - writeIcConfig(c.config, cfgArtifact) - result.add "--icPreparsedConfig:" & cfgArtifact + # Precompiled config: every child replays the one artifact produced (in a + # separate `nim icconfig` process) and already replayed by the driver itself — + # see `icconfig.ensureIcConfig`, run before the driver's own `loadConfigs`. So + # `nim ic` is always governed by this single artifact, for speed and so the + # driver and its children agree by construction. Forward the path the driver + # replayed (`conf.icPreparsedConfig`); `commandIc` has already guaranteed it + # exists, else it bailed. + result.add "--icPreparsedConfig:" & c.config.icPreparsedConfig proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): string = ## Frontend build file: the nifler (parse) and `nim m` (sem) rules only. The @@ -1029,6 +1040,12 @@ proc commandIc*(conf: ConfigRef) = rawMessage(conf, errGenerated, "nifler tool not found. Install nimony or add nifler to PATH.") return + # Resolve the `.nim` source first, exactly like `wantMainModule`. Without + # this, an extensionless project arg (`nim ic path/to/foo`) resolves to a + # same-named sibling that already exists — e.g. the ELF a prior `nim c` + # left behind — and nifler chokes on the binary (`invalid token \127`, + # ELF magic). `addFileExt` only appends when there is no extension. + conf.projectFull = addFileExt(conf.projectFull, NimExt) let projectFile = conf.projectFull.string if not fileExists(projectFile): rawMessage(conf, errGenerated, "project file not found: " & projectFile) @@ -1118,6 +1135,15 @@ proc commandIc*(conf: ConfigRef) = # from its importer — and rerun; nifmake's mtime pruning keeps completed # work. A round that discovers nothing new but still fails is a real error. let forwardedArgs = computeForwardedArgs(c) + # The precompiled config drives every `nim m`/`nim nifc` child and the driver + # itself (`ensureIcConfig` produced it and `loadConfigs` replayed it). If it + # is not on disk something went wrong producing it — children would each + # silently fall back to re-parsing the whole config chain — so refuse to + # continue without it. + if conf.icPreparsedConfig.len == 0 or not fileExists(conf.icPreparsedConfig): + rawMessage(conf, errGenerated, + "precompiled config missing: " & conf.icPreparsedConfig) + return let nifmake = findNifmake() # Build the per-module rules concurrently: nifmake fans out all commands at # each DAG depth via execProcesses (defaults to all cores). Cold builds are @@ -1164,8 +1190,9 @@ proc commandIc*(conf: ConfigRef) = let newNode = Node(files: @[pair], id: c.nodes.len) if c.systemNodeId >= 0: newNode.deps.add c.systemNodeId - for impId in c.implicitNodeIds: - if impId != newNode.id: newNode.deps.add impId + if getsImplicitImports(c, pair.nimFile): + for impId in c.implicitNodeIds: + if impId != newNode.id: newNode.deps.add impId c.processedModules[pair.modname] = newNode.id c.nodes.add newNode idx = newNode.id diff --git a/compiler/icconfig.nim b/compiler/icconfig.nim index 653fb2dfd6..0250ee0583 100644 --- a/compiler/icconfig.nim +++ b/compiler/icconfig.nim @@ -31,23 +31,39 @@ ## `--path` arguments; replaying their raw, config-dir-relative arguments here ## would misresolve. -import options, commands, lineinfos -import std/[algorithm, os, sets] +import options, commands, lineinfos, pathutils, msgs +import std/[algorithm, os, sets, osproc, times, streams, syncio] import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse] const - IcConfigVersion* = "1" + IcConfigVersion* = "2" ## Artifact format version. Bump on any layout change here so a child built ## by an older compiler rejects a stale artifact and falls back to normal ## config loading instead of replaying a format it cannot parse. proc writeIcConfig*(conf: ConfigRef; outfile: string) = - ## Serialise the config-file switches recorded during `loadConfigs` plus the - ## resolved `cppDefines` set into the artifact at `outfile`. - var b = nifbuilder.open(outfile) + ## Serialise the resolved config (the config-file switches recorded during + ## `loadConfigs`, the resolved `cppDefines`/`searchPaths`, the nimcache dir, and + ## the list of config *source* files for staleness detection) into `outfile`. + ## `OnlyIfChanged`: when the content is byte-identical to what is already on + ## disk the file is left untouched so its mtime does not advance — otherwise + ## every `nim ic` run would re-fire the whole nifmake graph (see `nifler`'s + ## `produceConfig`, whose model this mirrors). + var b = nifbuilder.open(outfile, writeMode = OnlyIfChanged) b.withTree "stmts": b.withTree "meta": b.addStrLit IcConfigVersion + b.withTree "sources": + # Every config file read while loading (nim.cfg chain + config.nims), so a + # later run can decide via mtimes whether this artifact is still current + # (see `sourcesChanged`). + for f in conf.configFiles: + b.addStrLit f.string + b.withTree "nimcache": + # Resolved build nimcache. Recorded (unlike the path-search switches) so the + # driver, which replays this artifact instead of parsing `nim.cfg`, still + # learns a `--nimcache:` set inside `nim.cfg` and builds in the right place. + b.addStrLit conf.nimcacheDir.string b.withTree "cppdefines": # HashSet iteration order is unspecified; sort so the artifact is # byte-stable across runs (nifmake keys rebuilds off content changes). @@ -55,6 +71,15 @@ proc writeIcConfig*(conf: ConfigRef; outfile: string) = for d in conf.cppDefines: defs.add d sort defs for d in defs: b.addStrLit d + b.withTree "searchpaths": + # The resolved (absolute) search paths. Path-search *switches* are skipped + # below because their raw arguments are config-dir-relative; the net effect + # lives here instead, so a replayer with no `--path` command-line arguments + # (the `nim ic` driver itself) still resolves imports. `nim m`/`nim nifc` + # children also receive these as forwarded `--path` args; the dedup on + # replay makes the overlap harmless. + for p in conf.searchPaths: + b.addStrLit p.string b.withTree "switches": for sw in conf.icConfigSwitches: b.addTree "sw" @@ -74,7 +99,10 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool = let stmtsTag = tags.registerTag("stmts") metaTag = tags.registerTag("meta") + sourcesTag = tags.registerTag("sources") + nimcacheTag = tags.registerTag("nimcache") cppTag = tags.registerTag("cppdefines") + pathsTag = tags.registerTag("searchpaths") switchesTag = tags.registerTag("switches") swTag = tags.registerTag("sw") var buf = parseFromFile(infile, 1000, pool, tags) @@ -95,6 +123,22 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool = inc c else: skip c + elif c.cursorTagId == nimcacheTag: + c.loopInto: + if c.kind == StrLit: + let nc = strVal(c) + # Only when nimcache was not already pinned on the command line: a + # `--nimcache:` argument the driver/child was launched with must win + # over whatever `nim.cfg` recorded into the artifact. + if nc.len > 0 and conf.nimcacheDir.isEmpty: + conf.nimcacheDir = AbsoluteDir(nc) + inc c + else: + skip c + elif c.cursorTagId == sourcesTag: + # Replay does not need the source list; it exists only for + # `sourcesChanged`. Skip the whole section. + skip c elif c.cursorTagId == cppTag: c.loopInto: if c.kind == StrLit: @@ -102,6 +146,17 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool = inc c else: skip c + elif c.cursorTagId == pathsTag: + c.loopInto: + if c.kind == StrLit: + # Append preserving the serialised order (which already reflects the + # driver's addPath insert-at-front sequence), deduping against any + # path a child already received via a forwarded `--path` argument. + let d = AbsoluteDir(strVal(c)) + if not conf.searchPaths.contains(d): conf.searchPaths.add d + inc c + else: + skip c elif c.cursorTagId == switchesTag: c.loopInto: if c.kind == TagLit and c.cursorTagId == swTag: @@ -125,3 +180,104 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool = skip c endRead(c) result = sawMeta and version == IcConfigVersion + +proc sourcesChanged*(configFile: string): bool = + ## True when the precompiled config at `configFile` is missing, malformed, + ## written by an incompatible version, or any recorded config *source* file is + ## newer than it (or has vanished) — i.e. the artifact must be regenerated. + ## Mirrors nifler's `sourcesChanged`: the source list lives inside the artifact + ## so this needs no out-of-band knowledge of which `nim.cfg`s were read. + if not fileExists(configFile): return true + let modtime = getLastModificationTime(configFile) + var pool = newPool() + var tags = newTagPool() + let + stmtsTag = tags.registerTag("stmts") + metaTag = tags.registerTag("meta") + sourcesTag = tags.registerTag("sources") + var buf = parseFromFile(configFile, 1000, pool, tags) + var c = beginRead(buf) + if c.kind != TagLit or c.cursorTagId != stmtsTag: + endRead(c) + return true + var version = "" + var depsChanged = false + c.loopInto: + if c.kind == TagLit and c.cursorTagId == metaTag: + c.loopInto: + if c.kind == StrLit: + version = strVal(c) + inc c + else: + skip c + elif c.kind == TagLit and c.cursorTagId == sourcesTag: + c.loopInto: + if c.kind == StrLit: + let dep = strVal(c) + if not fileExists(dep) or getLastModificationTime(dep) >= modtime: + depsChanged = true + inc c + else: + skip c + else: + skip c + endRead(c) + result = depsChanged or version != IcConfigVersion + +proc produceIcConfig*(conf: ConfigRef) = + ## The `cmdIcConfig` command. By the time it runs, the normal pipeline has + ## already fully parsed the `nim.cfg` chain and run `config.nims`, so the + ## resolved config is sitting in `conf`; just serialise it to `--o`. + let outPath = conf.icConfigOut + if outPath.len == 0: + rawMessage(conf, errGenerated, "icconfig: missing output path (--icConfigOut)") + return + createDir(parentDir(outPath)) + writeIcConfig(conf, outPath) + +proc ensureIcConfig*(conf: ConfigRef) = + ## Driver-side (`cmdIc`). Make sure an up-to-date precompiled config exists, + ## (re)producing it in a *separate* process when missing or stale, then point + ## `conf.icPreparsedConfig` at it so the driver replays the very same config its + ## `nim m`/`nim nifc` children will — perfect speed (config parsed at most once, + ## skipped entirely when nothing changed) and consistency (one producer, every + ## process replays its output). The artifact lives in the nimcache derived from + ## the command line (pre-config-parse), which is the one the children are told; + ## a `--nimcache:` set inside `nim.cfg` is recovered from the artifact itself. + let cacheDir = getNimcacheDir(conf).string + # Start from a clean cache when the on-disk NIF format stamp is absent or stale + # (see `icFormatVersion`). This must happen HERE, before the config artifact is + # produced — `commandIc` performs the same check later, but by then the artifact + # would already live in the cache and the wipe would delete it. + createDir(cacheDir) + let versionFile = cacheDir / "ic.version" + let stamp = if fileExists(versionFile): readFile(versionFile) else: "" + if stamp != icFormatVersion: + removeDir(cacheDir) + createDir(cacheDir) + writeFile(versionFile, icFormatVersion) + let outPath = cacheDir / "ic_config.cfg.nif" + if not fileExists(outPath) or sourcesChanged(outPath): + createDir(cacheDir) + # Re-invoke ourselves as the config producer: reuse this process's command + # line, dropping the command argument (`ic`) in favour of `icconfig` and the + # explicit output path, both BEFORE the project file (anything after the + # project is swallowed into `config.arguments` by `cmdLineRest`). The + # producer re-reads `nim.cfg` itself. + var pargs = @["icconfig", "--icConfigOut:" & outPath] + var droppedCmd = false + for a in commandLineParams(): + if not droppedCmd and a.len > 0 and a[0] != '-': + droppedCmd = true # drop the original command token (`ic`) + else: + pargs.add a + let p = startProcess(getAppFilename(), args = pargs, + options = {poStdErrToStdOut}) + let outp = p.outputStream.readAll() + let code = p.waitForExit() + p.close() + if code != 0 or not fileExists(outPath): + rawMessage(conf, errGenerated, + "failed to produce precompiled config (exit code " & $code & "):\n" & outp) + return + conf.icPreparsedConfig = outPath diff --git a/compiler/main.nim b/compiler/main.nim index bf7eb2b675..9052287a1f 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -29,6 +29,7 @@ when defined(nimPreviewSlimSystem): import ../dist/checksums/src/checksums/sha1 import pipelines +from icconfig import produceIcConfig when not defined(nimKochBootstrap): import nifbackend @@ -439,6 +440,11 @@ proc mainCommand*(graph: ModuleGraph) = commandIc(conf) else: rawMessage(conf, errGenerated, "nim deps not available in bootstrap build") + of cmdIcConfig: + # Produce the precompiled config artifact for `nim ic` (config already + # parsed by the normal pipeline); a separate process spawned by the driver. + wantMainModule(conf) + produceIcConfig(conf) of cmdParse: wantMainModule(conf) discard parseFile(conf.projectMainIdx, cache, conf) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index e145214bcb..af5df92902 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -1063,7 +1063,16 @@ proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym = proc belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool = ## Check if symbol belongs to the 'stdlib' package. - sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId + # Compare the package *name* (an interned ident), not the package symbol's + # `.id`. Under per-module IC (`nim m`) the system module is loaded from a NIF + # in a process that does not compile it from source, so its package symbol is + # reconstructed with a fresh `.id` that no longer matches the freshly-interned + # package of a stdlib module compiled standalone here — making the old id + # comparison wrongly report `false` and inject `--import`ed modules into the + # stdlib. Both are canonically named `stdlib` (lib/stdlib.nimble); in a normal + # `nim c` build (system compiled from source) the ids match too, so this is a + # no-op there. + sym.getPackageSymbol.name.id == graph.systemModule.getPackageSymbol.name.id proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): SuggestFileSymbolDatabase = result = graph.suggestSymbols.getOrDefault(fileIdx, newSuggestFileSymbolDatabase(fileIdx, optIdeExceptionInlayHints in graph.config.globalOptions)) diff --git a/compiler/nim.nim b/compiler/nim.nim index fd33aca040..0d2a0a95a4 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -29,6 +29,7 @@ import pathutils, modulegraphs from ast2nif import registerNifAstTags +from icconfig import ensureIcConfig from std/browsers import openDefaultBrowser from nodejs import findNodeJs @@ -114,6 +115,14 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = self.processCmdLineAndProjectPath(conf) + # `nim ic` driver: ensure the precompiled config exists (produced by a separate + # `nim icconfig` process, skipped when nothing changed) BEFORE config loading, + # so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the + # driver runs on the exact same config its children will. See icconfig.nim. + when not defined(nimKochBootstrap): + if conf.cmd == cmdIc: + ensureIcConfig(conf) + var graph = newModuleGraph(cache, conf) if not self.loadConfigsAndProcessCmdLine(cache, conf, graph): return diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 9c5e4c6d07..fc5f4e3656 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -246,10 +246,14 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = setDefaultLibpath(conf) - # `nim ic` children replay the precompiled config the driver recorded once, - # instead of re-reading the `nim.cfg` chain and re-running `config.nims` in the - # VM. A missing/format-incompatible artifact returns false: fall through to - # normal config loading so an older child or a deleted cache still works. + # The `nim ic` driver and its `nim m`/`nim nifc` children replay the precompiled + # config (produced once by a separate `nim icconfig` process — see + # `icconfig.ensureIcConfig`, which sets `icPreparsedConfig` for the driver + # before this runs; the children get it as a forwarded `--icPreparsedConfig` + # argument) instead of re-reading the `nim.cfg` chain and re-running + # `config.nims` in the VM. A missing/format-incompatible artifact returns false: + # fall through to a normal parse (this is also the path the `nim icconfig` + # producer itself takes, since it runs with no `icPreparsedConfig`). if conf.icPreparsedConfig.len > 0 and applyIcConfig(conf, conf.icPreparsedConfig): return template readConfigFile(path) = diff --git a/compiler/options.nim b/compiler/options.nim index 012f633946..8559caf0e3 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -200,6 +200,7 @@ type cmdCompileToNif cmdNifC # generate C code from NIF files cmdIc # generate .build.nif for nifmake + cmdIcConfig # `nim ic`'s precompiled-config producer (writes ic_config.cfg.nif) const cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, @@ -418,12 +419,16 @@ type # module's package the "main package" and unfilter # foreign-package diagnostics; the real project # restores whole-program filtering semantics. - icPreparsedConfig*: string # under `nim m`/`nim nifc`: path of the precompiled - # config artifact written once by the `nim ic` driver. + icPreparsedConfig*: string # under the `nim ic` driver and its `nim m`/`nim nifc` + # children: path of the precompiled config artifact. # When set, `loadConfigs` replays the recorded # config-file switches from it instead of re-reading # the `nim.cfg` chain and re-running `config.nims` - # (which the VM makes expensive) per subprocess. + # (which the VM makes expensive) per process. The + # artifact itself is produced by a separate + # `nim icconfig` process (see `cmdIcConfig`). + icConfigOut*: string # under `nim icconfig`: the path to write the + # precompiled config artifact to (set via `--o`). icConfigSwitches*: seq[tuple[switch, arg: string]] # the config-file (`passPP`) switches applied while # loading config, in order. Recorded by every nim diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 7c9ce97ff3..ae81ac12f1 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -286,6 +286,15 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF result = graph.getModule(fileIdx) template processModuleAux(moduleStatus) = + when defined(icDbg): + block: + let dbgf = open("/tmp/defdbg.txt", fmAppend) + dbgf.writeLine toFullPath(graph.config, fileIdx) & + " nimStackTraceOverride=" & $isDefined(graph.config, "nimStackTraceOverride") & + " nimscript=" & $isDefined(graph.config, "nimscript") & + " optCompress=" & $(optCompress in graph.config.globalOptions) & + " cmd=" & $graph.config.cmd + dbgf.close() onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) var s: PLLStream = nil if sfMainModule in flags: