From f3bdc6c5f228af0cda394b8570144fd7d3fc8981 Mon Sep 17 00:00:00 2001 From: araq Date: Wed, 26 Aug 2026 10:32:31 +0200 Subject: [PATCH] IC: `nim c --ic:on` / `nim cpp --ic:on` replace the `nim ic` command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `nim ic` was a command of its own, which made it the C backend only and cut it off from everything the ordinary compile commands accept. `--ic:on` is a switch on `nim c` / `nim cpp` / `nim objc` instead, so `-r`, `-d:release`, `--exceptions:`, and a project-wide opt-in from `nim.cfg` / `config.nims` all work. `nim ic` still resolves to the same driver; `koch bootic`, `koch ic` and `testament --ic` now go through the switch. The switch was already parsed into a `conf.ic` nobody read. It is now read in `passCmd1` as well, because `nim.nim` has to decide whether this run is an IC DRIVER before config loading (`ensureIcConfig` produces the precompiled config the driver itself replays); when the switch comes from a config file instead, `main.nim` produces it late. **C++.** `tests/cpp` passes under `--ic:on`, matching its classic result. Four fixes, three of them the shape of every "C++ needs the whole program" problem the per-module backend has: * The driver DECLARES each module's translation unit to nifmake without loading a module, so it cannot ask `cgen.getCFile` — and it hardcoded `.nim.c`, so the merge stage went looking for `.c.nif` next to the `.cpp.nif` the children had written. `options.icCFileExt` mirrors the formula at backend granularity. * C++ has no designated initializers, so the RTTI record is a bare variable that `DatInit` fills field by field. A bare `TNimTypeV2 x;` is a tentative definition — C's linker merges those, C++'s does not — so every TU that demanded the type defined it ("multiple definition of NTIv2__…"). It now gets the same extern-declaration + owned-`'d'`-definition split the C flavour has. * `memberProcsPerType` / `initializersPerType` live only in the sem process, so the backend emitted a struct WITHOUT its in-class member declarations and the out-of-class definitions did not match ("no declaration matches 'void Doo::memberProc()'"). They are replayed from a new `(repcppmember …)` log entry; `replayCppMember` re-derives the type from the routine's signature exactly as `semCppMember` does, so no type key has to survive the round trip. * Two follow-ons for members: `loc.snippet` is a CALL PATTERN (`#->salute(@)`), and only `genMemberProcHeader` derives it — whole-program cgen got it for free by generating the defining module first, but the per-module backend emits that body in another process, leaving the caller with the mangled Nim name (`loo->salute_u0__vireouyks1()`). And that pattern is not a linker name: every `salute` member in every class mints the same one, so the merge stage handed them all to one artifact and dropped the rest (undefined vtable at link). Member definitions are keyed by their NIF name there instead. `--run` is now dropped when re-invoking for the config artifact: the producer has no output binary and `nim.nim`'s run step asserted on the empty `outFile`. testament's `--ic` appends the switch rather than rewriting the compile verb, so a test that overrides `cmd:` wholesale keeps its verb, and the C++ corpus is covered too (it never was — the old rewrite only matched `nim c `). `icFormatVersion` 36 -> 37 for the new log entry. `koch bootic` reaches its byte-identical fixed point through the new entry point; `koch ic` passes; `tests/ic`, `tests/destructor` (3 known `--newruntime` failures) and the classic categories are unchanged. Co-Authored-By: Claude Opus 5 --- compiler/ast2nif.nim | 8 +++++ compiler/astdef.nim | 2 +- compiler/ccgtypes.nim | 35 +++++++++++++++--- compiler/cgen.nim | 30 +++++++++++++--- compiler/commands.nim | 9 +++-- compiler/deps.nim | 15 +++++++- compiler/icconfig.nim | 17 ++++++++- compiler/main.nim | 24 ++++++++++++- compiler/modulegraphs.nim | 47 +++++++++++++++++++++++- compiler/nifbackend.nim | 6 ++-- compiler/nim.nim | 2 +- compiler/options.nim | 20 ++++++++++- compiler/semstmts.nim | 2 ++ doc/ic.md | 75 ++++++++++++++++++++++++++++++--------- koch.nim | 6 ++-- testament/testament.nim | 48 ++++++++++++------------- 16 files changed, 282 insertions(+), 64 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 8bad196801..f6d2094090 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -1324,6 +1324,7 @@ var repDeepCopyTag = registerTag("repdeepcopy") var repEnumToStrTag = registerTag("repenumtostr") var repMethodTag = registerTag("repmethod") var repPureEnumTag = registerTag("reppureenum") +var repCppMemberTag = registerTag("repcppmember") #var repClassTag = registerTag("repclass") var includeTag = registerTag("include") var importTag = registerTag("import") @@ -1401,6 +1402,7 @@ proc registerNifAstTags*() = repEnumToStrTag = registerTag("repenumtostr") repMethodTag = registerTag("repmethod") repPureEnumTag = registerTag("reppureenum") + repCppMemberTag = registerTag("repcppmember") includeTag = registerTag("include") importTag = registerTag("import") implTag = registerTag("implementation") @@ -1719,6 +1721,11 @@ proc writeOp(w: var Writer; content: var IcBuilder; op: LogEntry) = content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() + of CppMemberEntry: + content.addParLe repCppMemberTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) + content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) + content.addParRi() of GenericInstEntry: discard "will only be written later to ensure it is materialized" @@ -3832,6 +3839,7 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag] elif tagIs(cur, "repenumtostr"): loadLogOp(c, result.logOps, cur, EnumToStrEntry, attachedTrace, module) elif tagIs(cur, "repmethod"): loadLogOp(c, result.logOps, cur, MethodEntry, attachedTrace, module) elif tagIs(cur, "reppureenum"): loadLogOp(c, result.logOps, cur, PureEnumEntry, attachedTrace, module) + elif tagIs(cur, "repcppmember"): loadLogOp(c, result.logOps, cur, CppMemberEntry, attachedTrace, module) elif tagIs(cur, "export"): cur.into: while cur.hasMore and cur.kind == DotToken: skip cur # flags / type diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 11f9a4474d..42ccf4c7b0 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -1049,7 +1049,7 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode = type LogEntryKind* = enum HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry, - PureEnumEntry + PureEnumEntry, CppMemberEntry LogEntry* = object kind*: LogEntryKind op*: TTypeAttachedOp diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index a7416e0815..08cab8c13b 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1289,6 +1289,14 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool name = typDesc if isFnConst: fnConst = " const" + if not isCtor: + # The call-site form (`x->salute(@)`), not the mangled Nim name. Set it on + # BOTH paths: whole-program cgen always emitted the out-of-class definition + # (the `else` branch) before any caller, but the per-module backend emits a + # foreign member proc's body in ITS OWN module, so the caller's TU only ever + # reaches the in-class declaration below — and called the member by the + # mangled name (`loo->salute_u0__vireouyks1()`, "struct Loo has no member"). + prc.locImpl.snippet = "$1$2(@)" % [memberOp, name] if isFwdDecl: if isStatic: result.add "static " @@ -1298,9 +1306,7 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool override = " override" superCall = "" else: - if not isCtor: - prc.locImpl.snippet = "$1$2(@)" % [memberOp, name] - elif superCall != "": + if isCtor and superCall != "": superCall = " : " & superCall name = "$1::$2" % [typDesc, name] @@ -1891,11 +1897,30 @@ proc genVTable(result: var Builder, seqs: seq[PSym]) = result.add(cCast(CPointer, seqs[i].loc.snippet)) proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) = + ## The C++/HCR flavour: C++ has no designated initializers, so the RTTI record + ## is a bare variable that the module's `DatInit` fills field by field. cgsym(m, "TNimTypeV2") - m.s[cfsStrData].addDeclWithVisibility(Private): - m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2") if m.config.cmd == cmdNifC: + # Same emit-everywhere split as `genTypeInfoV2Impl`: every `cg` process that + # demands this type declares it `extern`, and the DEFINITION is a droppable + # `'d'` unit the merge stage gives a single owner. Without the split the bare + # `TNimTypeV2 x;` in each TU is a tentative definition — which C's linker + # merges but C++'s does not, so `nim cpp --ic:on` died at link with + # "multiple definition of NTIv2__…". The field ASSIGNMENTS stay in every + # TU's `DatInit`: they are top-level code, not a definition, and every module + # computes the same values. + m.s[cfsStrData].addDeclWithVisibility(Extern): + m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2") + m.s[cfsVars].add(cnifDefDirective(name, "d", icNifName(m, origType))) + var def = newBuilder("") + def.addDeclWithVisibility(Private): + def.addVar(kind = Local, name = name, typ = "TNimTypeV2") + m.s[cfsVars].add extract(def) + m.s[cfsVars].add(cnifEndDefs()) m.icDataDefs.add (name, icNifName(m, origType)) + else: + m.s[cfsStrData].addDeclWithVisibility(Private): + m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2") var flags = 0 if not canFormAcycle(m.g.graph, t): flags = flags or 1 diff --git a/compiler/cgen.nim b/compiler/cgen.nim index cf48f9dc73..4250575c09 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1635,8 +1635,17 @@ proc genProcLvl3*(m: BModule, prc: PSym) = # `extern`/`rtl` pragma at sem time), so its uses are invisible to the # artifact's liveness walk — conservatively keep the definition. defFlags.add 'x' - m.s[cfsProcs].add(cnifDefDirective(stripCnifMarks(prc.loc.snippet), defFlags, - icNifName(m, prc))) + # A C++ member's `loc.snippet` is a CALL PATTERN (`#->salute(@)`), not a + # linker name — and every member of that name, in every class, mints the + # same one. Ownership is assigned per name, so `Loo::salute` and `Foo::salute` + # collided: the merge stage handed both to one artifact and the other TU's + # definition was dropped (undefined vtable at link). Key member definitions by + # their NIF name instead, which is unique by construction. Dots cannot occur + # in a mangled C name, so the two namespaces stay disjoint. + let defName = + if sfCppMember * prc.flags != {}: icNifName(m, prc) + else: stripCnifMarks(prc.loc.snippet) + m.s[cfsProcs].add(cnifDefDirective(defName, defFlags, icNifName(m, prc))) m.s[cfsProcs].add(extract(generatedProc)) m.s[cfsProcs].add(cnifEndDefs()) else: @@ -1661,7 +1670,20 @@ proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} = proc genProcPrototype(m: BModule, sym: PSym) = useHeader(m, sym) - if lfNoDecl in sym.loc.flags or sfCppMember * sym.flags != {}: return + if lfNoDecl in sym.loc.flags: return + if sfCppMember * sym.flags != {}: + # A C++ member is declared INSIDE its class, never as a free prototype — but + # this TU still needs its CALL-SITE name (`x->salute(@)`), and only + # `genMemberProcHeader` derives that (from the pragma's declaration pattern). + # Whole-program cgen got it for free: the module defining the member was code + # generated in the same process, ahead of any caller. The per-module backend + # emits that body in ANOTHER process, so the caller was left with the mangled + # Nim name `fillBackendName` minted and C++ rejected + # `loo->salute_u0__vireouyks1()` ("struct Loo has no member named ..."). + if m.compileToCpp: + var scratch = newBuilder("") + genMemberProcHeader(m, sym, scratch, false, true) + return if lfDynamicLib in sym.loc.flags: if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg": # Under IC per-module cg every demander emits the dynlib proc's DEFINITION @@ -2711,7 +2733,7 @@ proc getCFile*(m: BModule): AbsoluteFile = let ext = if m.compileToCpp: ".nim.cpp" elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m" - else: ".nim.c" + else: icCFileExt(m.config) result = changeFileExt(completeCfilePath(m.config, mangleModuleName(m.config, m.cfilename).AbsoluteFile), ext) when false: diff --git a/compiler/commands.nim b/compiler/commands.nim index 1182a4a377..680ab8427e 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1092,9 +1092,14 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) helpOnError(conf, pass) of "symbolfiles", "incremental", "ic": - if switch.normalize == "symbolfiles": deprecatedAlias(switch, "incremental") + if pass in {passCmd2, passPP} and switch.normalize == "symbolfiles": + deprecatedAlias(switch, "incremental") # xxx maybe also ic, since not in help? - if pass in {passCmd2, passPP}: + # `--ic:on` is read in passCmd1 too: `nim.nim` decides BEFORE config loading + # whether this run is an IC driver (`ensureIcConfig` must produce the + # precompiled config the driver itself then replays), and passCmd1 is the + # only pass that has run by then. + if pass in {passCmd1, passCmd2, passPP}: case arg.normalize of "on": conf.ic = true of "legacy": conf.symbolFiles = v2Sf diff --git a/compiler/deps.nim b/compiler/deps.nim index a812d949b2..9450670edf 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -934,6 +934,19 @@ proc computeForwardedArgs(c: DepContext): seq[string] = # them — phantom outputs that re-fire the build on every rerun). if c.config.selectedGC != gcUnselected: result.add "--mm:" & $c.config.selectedGC + # The children are invoked as `nim m` / `nim nifc`, so the driver's own command + # token (`c`, `cpp`, `ic`) is gone and with it the backend it selected. Name it + # explicitly — `nim cpp --ic:on` must not have its stdlib sem'd and its TUs + # emitted as C. The exception model rides along for the same reason: `nim cpp` + # defaults to `--exceptions:cpp`, which changes both codegen and sem. + if c.config.backend != backendInvalid: + result.add "--backend:" & $c.config.backend + if c.config.exc != excNone: + result.add "--exceptions:" & (case c.config.exc + of excGoto: "goto" + of excCpp: "cpp" + of excQuirky: "quirky" + else: "setjmp") # method dispatch semantics must match across the child processes: # a child compiled without --multimethods:on builds different dispatch # buckets (and rejects calls as ambiguous that multi-dispatch accepts) @@ -1232,7 +1245,7 @@ proc backendCFile(c: DepContext; node: Node): string = if node.id == 0: AbsoluteFile node.files[0].nimFile else: AbsoluteFile node.files[0].modname result = changeFileExt(completeCfilePath(c.config, - mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string + mangleModuleName(c.config, cfilename).AbsoluteFile), icCFileExt(c.config)).string proc computeLiveBackendNodes(c: DepContext): seq[bool] = ## Which nodes the backend must code-generate: the closure reachable from the diff --git a/compiler/icconfig.nim b/compiler/icconfig.nim index a0255c23dc..55c8ad8815 100644 --- a/compiler/icconfig.nim +++ b/compiler/icconfig.nim @@ -32,7 +32,7 @@ ## would misresolve. import options, commands, lineinfos, pathutils, msgs -import std/[algorithm, os, sets, osproc, times, streams, syncio] +import std/[algorithm, os, sets, osproc, times, streams, syncio, strutils] import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse] const @@ -269,11 +269,26 @@ proc ensureIcConfig*(conf: ConfigRef) = # verbatim: all `-`-prefixed switches first (in encounter order), then the # non-switch project token(s). The producer re-reads `nim.cfg` itself. var pargs = @["icconfig", "--icConfigOut:" & outPath] + # The command token is dropped below, so `nim cpp --ic:on` would hand the + # producer a C-backend config: name the backend explicitly. (`nim ic + # --backend:cpp` already carries the switch; the duplicate is harmless.) + if conf.backend != backendInvalid: + pargs.add "--backend:" & $conf.backend var rest: seq[string] = @[] var droppedCmd = false for a in commandLineParams(): if a.len == 0: continue if a[0] == '-': + # `--run`/`-r` must not reach the producer: it only serialises the + # resolved config, has no output binary, and `nim.nim`'s run step asserts + # on the empty `outFile` (`nim cpp --ic:on -r foo.nim`). + var name = "" + var i = 1 + if i < a.len and a[i] == '-': inc i + while i < a.len and a[i] notin {':', '='}: + name.add a[i] + inc i + if normalize(name) in ["r", "run"]: continue pargs.add a elif not droppedCmd: droppedCmd = true # drop the original command token (`ic`/`track`) diff --git a/compiler/main.nim b/compiler/main.nim index 0365eba486..cddf2fc96b 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -29,7 +29,7 @@ when defined(nimPreviewSlimSystem): import ../dist/checksums/src/checksums/sha1 import pipelines -from icconfig import produceIcConfig +from icconfig import produceIcConfig, ensureIcConfig when not defined(nimKochBootstrap): import nifbackend @@ -269,6 +269,28 @@ proc mainCommand*(graph: ModuleGraph) = proc compileToBackend() = customizeForBackend(conf.backend) + if isIcDriver(conf): + # `nim c --ic:on` / `nim cpp --ic:on`: same driver as `nim ic`, entered + # through the ordinary compile command so every backend switch the user + # already knows keeps working (`nim cpp`, `--exceptions:`, `-d:`, ...). + # `customizeForBackend` above has already defined the backend symbol and + # picked the exception model, which is exactly what the per-module + # children must inherit — `computeForwardedArgs` forwards both. + setUseIc(true) + wantMainModule(conf) + setOutFile(conf) + when not defined(nimKochBootstrap): + if conf.icPreparsedConfig.len == 0: + # `--ic:on` came from a `nim.cfg`/`config.nims` rather than the command + # line, so `nim.nim` could not see it before config loading and the + # precompiled config the children replay does not exist yet. Produce it + # now. (The driver then keeps the config IT parsed instead of replaying + # the artifact; both come from the same files.) + ensureIcConfig(conf) + commandIc(conf) + else: + rawMessage(conf, errGenerated, "--ic:on not available in bootstrap build") + return setOutFile(conf) case conf.backend of backendC: commandCompileToC(graph) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index a392c7020f..31ae39828b 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -490,6 +490,49 @@ proc logMethodDef*(g: ModuleGraph; s: PSym) = g.opsLog.add LogEntry(kind: MethodEntry, module: s.itemId.module.int, key: "", sym: s) +proc logCppMember*(g: ModuleGraph; s: PSym) = + ## Log a C++ `{.member.}`/`{.virtual.}`/`{.constructor.}` registration (and the + ## `importcpp` default-initializer flavour) so the NIF backend can rebuild + ## `memberProcsPerType`/`initializersPerType`, which live only in the sem + ## process. Without them the per-module backend emitted the struct WITHOUT its + ## in-class member declarations and the out-of-class definitions did not match + ## ("no declaration matches 'void Doo::memberProc()'"). + ## + ## No type key: `replayCppMember` re-derives the type from the routine's + ## signature exactly as `semCppMember` does, so nothing has to survive the + ## round trip except the routine itself. + if g.config.cmd in {cmdNifC, cmdM}: + g.opsLog.add LogEntry(kind: CppMemberEntry, module: s.itemId.module.int, + key: "", sym: s) + +proc replayCppMember*(g: ModuleGraph; s: PSym) = + ## Inverse of `logCppMember`, mirroring `semstmts.semCppMember`'s derivation. + if s == nil or s.typ == nil: return + if sfImportc notin s.flags: + var typ = if sfConstructor in s.flags: s.typ.returnType else: s.typ.firstParamType + if typ != nil and typ.kind == tyPtr and sfConstructor notin s.flags: + typ = typ.elementType + if typ != nil and typ.kind == tyObject: + let procs = addr g.memberProcsPerType.mgetOrPut(typ.bindingId, @[]) + for prc in procs[]: + if prc == s: return + procs[].add s + else: + let typ = s.typ.returnType + if typ != nil and typ.kind == tyObject and + typ.bindingId notin g.initializersPerType and s.typ.n != nil: + # The default values sem read off the `nkIdentDefs` live on the param syms. + var call = newTree(nkCall, newSymNode(s)) + var isInitializer = s.typ.n.len > 1 + for i in 1 ..< s.typ.n.len: + let p = s.typ.n[i] + if p.kind != nkSym or p.sym.ast == nil or p.sym.ast.kind == nkEmpty: + isInitializer = false + break + call.add p.sym.ast + if isInitializer: + g.initializersPerType[typ.bindingId] = call + proc registerLoadedMethod*(g: ModuleGraph; m: PSym) = ## Rebuild the dispatch buckets from a serialized method registration. ## Buckets group the methods sharing a dispatcher; the dispatcher's BODY @@ -972,6 +1015,8 @@ when not defined(nimKochBootstrap): g.loadedOps[x.op][x.key] = x.sym of EnumToStrEntry: g.loadedEnumToStringProcs[x.key] = x.sym + of CppMemberEntry: + replayCppMember(g, x.sym) of MethodEntry: # only `methodDef` registrations (empty key) rebuild dispatch # buckets; the `addMethodToGeneric` flavor (typeKey key) announces @@ -1162,7 +1207,7 @@ when not defined(nimKochBootstrap): discard "dispatch buckets already rebuilt by registerLoadedHooks" of GenericInstEntry: raiseAssert "GenericInstEntry should not be in the NIF index" - of HookEntry, EnumToStrEntry: + of HookEntry, EnumToStrEntry, CppMemberEntry: discard "already done by registerLoadedHooks" # Register methods per type from NIF index discard "todo" diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index f4b1ce2e20..91ec2774e6 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -750,7 +750,7 @@ proc generateMergeStage(g: ModuleGraph) = let p = line.strip() if p.len > 0: files.add p else: - for artifact in walkFiles(nimcache / "*.c.nif"): + for artifact in walkFiles(nimcache / ("*" & icCFileExt(g.config) & ".nif")): files.add artifact sort files let decision = computeMergeDecision(files) @@ -790,7 +790,7 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) = if targetIsMain: AbsoluteFile toFullPath(g.config, mainFileIdx) else: AbsoluteFile g.config.icBackendModule let cfile = changeFileExt(completeCfilePath(g.config, - mangleModuleName(g.config, cfilename).AbsoluteFile), ".nim.c").string + mangleModuleName(g.config, cfilename).AbsoluteFile), icCFileExt(g.config)).string let artifact = cfile & ".nif" if not fileExists(artifact): rawMessage(g.config, errGenerated, @@ -873,7 +873,7 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) = if not decision.broken: var liveOwners = initHashSet[string]() for cname, owner in decision.owners: - if owner.endsWith(".c.nif") and cname in decision.live: + if owner.endsWith(icCFileExt(g.config) & ".nif") and cname in decision.live: liveOwners.incl owner for owner in liveOwners: let cbase = owner[0 ..< owner.len - ".nif".len] # "@m….nim.c.nif" -> ".c" diff --git a/compiler/nim.nim b/compiler/nim.nim index 789657e216..68765365a2 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -120,7 +120,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = # so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the # driver runs on the exact same config its children will. See icconfig.nim. when not defined(nimKochBootstrap): - if conf.cmd in {cmdIc, cmdTrack}: + if conf.cmd in {cmdIc, cmdTrack} or isIcDriver(conf): ensureIcConfig(conf) var graph = newModuleGraph(cache, conf) diff --git a/compiler/options.nim b/compiler/options.nim index 0b8c06fbe8..a8a874a7f5 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -29,7 +29,7 @@ const nimEnableCovariance* = defined(nimEnableCovariance) - icFormatVersion* = "36" + icFormatVersion* = "37" ## Version of the IC cache format (the sem-NIF module layout written by ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever ## that layout changes: `commandIc` wipes a nimcache whose `ic.version` @@ -940,6 +940,24 @@ proc getOsCacheDir(): string = else: result = getHomeDir() / genSubDir.string +proc isIcDriver*(conf: ConfigRef): bool = + ## True for `nim c --ic:on` / `nim cpp --ic:on`: this process is the `nim ic` + ## DRIVER (it builds the nifmake graph and spawns the per-module children), + ## not a compilation. `nim ic` itself keeps its own `cmdIc` branch. + conf.ic and conf.cmd in {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC} + +proc icCFileExt*(conf: ConfigRef): string = + ## The extension the per-module backend gives a module's translation unit. + ## Mirrors `cgen.getCFile` at BACKEND granularity, which is all the `nim ic` + ## driver can know: it DECLARES every module's `.c`/`.cpp` output to nifmake + ## without loading a single module, so a per-module `{.compile: cpp.}` + ## (`sfCompileToCpp`) is out of reach — and `nim cpp` selects the backend for + ## the whole program anyway. + case conf.backend + of backendCpp: ".nim.cpp" + of backendObjc: ".nim.m" + else: ".nim.c" + proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir = proc nimcacheSuffix(conf: ConfigRef): string = if conf.ideActive: "_nimsuggest" # dedicated cache, never shared with `nim c` diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 9b9e8a75b9..6f9696cea6 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2410,6 +2410,7 @@ proc semCppMember(c: PContext; s: PSym; n: PNode) = localError(c.config, n.info, pragmaName & " must be either ptr to object or object type.") if sameOwners(typ.owner, s.owner) and sameOwners(c.module, s.owner): c.graph.memberProcsPerType.mgetOrPut(typ.bindingId, @[]).add s + logCppMember(c.graph, s) else: localError(c.config, n.info, pragmaName & " procs must be defined in the same scope as the type they are virtual for and it must be a top level scope") @@ -2432,6 +2433,7 @@ proc semCppMember(c: PContext; s: PSym; n: PNode) = inc j if isInitializer: c.graph.initializersPerType[typ.bindingId] = initializerCall + logCppMember(c.graph, s) proc semMethodPrototype(c: PContext; s: PSym; n: PNode) = if s.isGenericRoutine: diff --git a/doc/ic.md b/doc/ic.md index 162b7bbd3e..a6e6860597 100644 --- a/doc/ic.md +++ b/doc/ic.md @@ -2,12 +2,23 @@ Incremental Compilation (IC) ====================================== -The ``nim ic`` command provides incremental compilation for Nim projects. It -decomposes compilation into per-module steps whose results are cached as NIF -files, and uses the external ``nifmake`` build tool to re-run only the steps -whose inputs changed. +``--ic:on`` turns an ordinary compile into an incremental one. It decomposes +compilation into per-module steps whose results are cached as NIF files, and +uses the external ``nifmake`` build tool to re-run only the steps whose inputs +changed. -This document describes **how `nim ic` works today**, including the edge cases +.. code-block:: cmd + + nim c --ic:on myproject.nim + nim cpp --ic:on myproject.nim + +It is a switch on the normal compile commands, not a command of its own, so +everything else keeps working unchanged: ``cpp`` and ``objc`` backends, ``-r``, +``-d:release``, ``--exceptions:``, and a project-wide opt-in from ``nim.cfg`` / +``config.nims``. The older spelling ``nim ic`` still works and drives the same +code, but it is the C backend only and cannot run the binary it built. + +This document describes **how IC works today**, including the edge cases that shaped the current design. The per-module backend rewrite that earlier editions of this document listed as a *Plan* has **landed**: the whole-program, reuse/redirect/def-retention backend is gone and codegen is now a set of @@ -16,7 +27,7 @@ reuse/redirect/def-retention backend is gone and codegen is now a set of Overview ======== -The pipeline has two halves driven by one process (`nim ic`, `commandIc` in +The pipeline has two halves driven by one process (the *driver*, `commandIc` in ``compiler/deps.nim``) that constructs a dependency graph, writes a build file, and hands it to ``nifmake``: @@ -220,7 +231,7 @@ Edge cases (and why the machinery exists) - **Config cost.** Each child re-parsing `nim.cfg` + re-running `config.nims` in the VM was ~80 ms; replaced by a precompiled `ic_config.cfg.nif` replayed in `loadConfigs` (`compiler/icconfig.nim`). -- **`koch bootic`** bootstraps the compiler through `nim ic` (a 3-iteration +- **`koch bootic`** bootstraps the compiler through `--ic:on` (a 3-iteration fixed-point check). It writes its binary to ``bin/nim_ic`` and never clobbers ``bin/nim``. @@ -247,7 +258,7 @@ Known residual hack Status and performance ====================== -`nim ic` self-builds the compiler (`koch bootic`'s byte-identical fixed-point +IC self-builds the compiler (`koch bootic`'s byte-identical fixed-point check) under both `orc` and `--mm:refc`, and passes the external-package CI set. Cold full bootstrap on a 32-core box (`-d:release`, **no edits** — IC's worst @@ -256,7 +267,7 @@ case, since incremental reuse is not exercised): | | wall | notes | | - | ---- | ----- | | `koch boot` (classic) | ~1m00s | reference | -| `koch bootic` (`nim ic`) | ~1m39s | **~1.66×** | +| `koch bootic` (`--ic:on`) | ~1m39s | **~1.66×** | This is down from ~7.5× in the whole-program-backend era. IC does modestly more aggregate work (more processes, NIF re-parsing of imports per process), but on a @@ -412,7 +423,7 @@ Testing IC Two mechanisms, at very different scales. **`tests/ic` — metamorphic tests.** A `t*.nim` whose body contains `#? metamorphic` -drives a sequence of cross-module edits through `nim ic` in one fixed build +drives a sequence of cross-module edits through the IC driver in one fixed build directory (see `testament/categories.nim`, `runMetamorphicIcTest`). Directives: | directive | effect | @@ -436,11 +447,13 @@ had nowhere to live on a serialized sym node, so every first assignment to a destructor-bearing local became `=sink` over zeroed memory). `koch bootic` has the same blind spot — it proves the compiler reproduces *itself*. -**`testament --ic` — the whole corpus.** Compiles every C-target test with -`nim ic` instead of `nim c`, so IC inherits the existing ~10k programs and their -expected output instead of the handful written for it by hand. Tests that -override the command (`cmd: "nim c --gc:arc $file"`) are rewritten too, and get a -private nimcache; without one they would share a cache and thrash it. +**`testament --ic` — the whole corpus.** Appends `--ic:on` to every C and C++ +test compile, so IC inherits the existing ~10k programs and their expected +output instead of the handful written for it by hand. Because it is a switch and +not a command, a test that overrides the command wholesale (`cmd: "nim cpp -r +$file"`) simply gains the switch — no verb rewriting, and the C++ corpus comes +along for free. Each also gets a private nimcache; without one they would share +a cache and thrash it. To keep that affordable, testament borrows nimony's hastur model (`warmupSharedCache` + `prefillFromWarmup`): a generated warmup program pulling in @@ -458,7 +471,7 @@ Measured on `tests/destructor` (97 test runs, 32-core box): | | cold | warm | | - | ---- | ---- | | `nim c` | 35s | 32s | -| `nim ic` | ~3m30 | **9.8s** | +| `--ic:on` | ~3m30 | **9.8s** | The warm number is the developer loop and it is 3.2x faster than the classic backend; the cold number is paid once per configuration and then cached on disk. @@ -471,3 +484,33 @@ hint, a warning — all of it is produced by the process that actually runs, so build that reuses every artifact prints nothing. Tests that check `nimout` (and anything you are debugging by eye) therefore need a cold cache; running the same test twice in a row makes the second run's `nimout` empty. + +The C++ backend +=============== + +``nim cpp --ic:on`` works, and `tests/cpp` passes under it. Three things had to +change for that, and they are worth knowing because they are the shape of every +"C++ needs the whole program" problem the per-module backend has: + +* **The driver must name the right file.** ``deps.nim`` DECLARES each module's + translation unit to ``nifmake`` without loading a single module, so it cannot + ask ``cgen.getCFile``; ``options.icCFileExt`` mirrors that formula at backend + granularity (``.nim.cpp`` / ``.nim.m`` / ``.nim.c``). + +* **C++ has no designated initializers**, so the RTTI record is a bare variable + that ``DatInit`` fills field by field. That bare ``TNimTypeV2 x;`` is a + tentative definition, which C's linker merges and C++'s does not — every TU + that demanded the type defined it. It now gets the same extern-declaration + + owned-``'d'``-definition split the C flavour has. + +* **A C++ member is declared inside its class.** ``memberProcsPerType`` and + ``initializersPerType`` live only in the sem process, so the backend emitted + the struct WITHOUT its member declarations; they are replayed from a + ``(repcppmember …)`` log entry now (``modulegraphs.replayCppMember`` re-derives + the type from the routine's signature, exactly as ``semCppMember`` does). + Two follow-on details: a member's ``loc.snippet`` is a CALL PATTERN + (``#->salute(@)``), so it must be computed even in the TU that only *calls* the + member (whole-program cgen got that for free by generating the defining module + first), and it is not a linker name — every ``salute`` member in every class + mints the same one, so definitions are keyed by their NIF name in the merge + stage instead. diff --git a/koch.nim b/koch.nim index 50d76e03d1..225956c639 100644 --- a/koch.nim +++ b/koch.nim @@ -76,7 +76,7 @@ Options: --skipIntegrityCheck skips integrity check when booting the compiler Possible Commands: boot [options] bootstraps with given command line options - bootic [options] bootstraps via the incremental compiler (`nim ic`) + bootic [options] bootstraps via the incremental compiler (`--ic:on`) distrohelper [bindir] helper for distro packagers tools builds Nim related tools toolsNoExternal builds Nim related tools (except external tools, @@ -450,7 +450,7 @@ proc bootic(args: string, skipIntegrityCheck: bool) = # everything. if i > 0: removeDir smartNimcache let nimi = if i == 0: nimStart else: i.thVersion - exec "$# ic --nimcache:$# $# compiler" / "nim.nim" % + exec "$# c --ic:on --nimcache:$# $# compiler" / "nim.nim" % [nimi, smartNimcache, args] if sameFileContent(output, i.thVersion): copyExe(output, finalDest) @@ -615,7 +615,7 @@ proc runIcTestFile(inp: string) = for fragment in content.split("#!EDIT!#"): let file = inp.replace(".nim", "_temp.nim") writeFile(file, fragment) - var cmd = nimExe & " ic --hint:Conf:off --warnings:off " + var cmd = nimExe & " c --ic:on --hint:Conf:off --warnings:off " cmd.add quoteShell(file) exec(cmd) diff --git a/testament/testament.nim b/testament/testament.nim index 478184fe28..bf2bb87c49 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -201,32 +201,31 @@ proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, var options = target.defaultOptions & ' ' & options if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) options.add ' ' & extraOptions - # `--ic` swaps the C target's command; every other target is left alone (the - # incremental compiler has a C backend only). - let targetCmd = - if useIc and target == targetC: "ic" else: targetToCmd[target] # we avoid using `parseCmdLine` which is buggy, refs bug #14343 - result = cmdTemplate % ["target", targetCmd, + result = cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, "filedir", filename.getFileDir(), "nim", compilerPrefix] - if useIc and target == targetC: - # Roughly half the corpus overrides the command wholesale (`cmd: "nim c - # --gc:arc $file"`), which neither goes through `$target` nor picks up - # `$options` — so those tests would silently keep using the classic backend - # and share one nimcache. Rewrite the compile verb and give them a private - # cache, which is what makes them incremental at all. - let prefix = compilerPrefix & " c " - if result.startsWith(prefix): - result = compilerPrefix & " ic " & result[prefix.len .. ^1] + if useIc and target in {targetC, targetCpp}: + # `--ic:on` turns the ordinary compile command into the IC driver, so the + # verb is left alone: roughly half the corpus overrides the command wholesale + # (`cmd: "nim c --gc:arc $file"`), which neither goes through `$target` nor + # picks up `$options`, and such a test now simply gains the switch. Each also + # gets a private nimcache, which is what makes it incremental at all. + # + # Switches must land BEFORE the project file: anything after it is swallowed + # into `config.arguments`, and a non-empty `arguments` without `--run` is a + # hard error ("arguments can only be given if the '--run' option is + # selected"). + var switches = "--ic:on " if nimcache.len > 0 and "--nimCache:" notin result and "--nimcache:" notin result: - # Must land BEFORE the project file: anything after it is swallowed into - # `config.arguments`, and a non-empty `arguments` without `--run` is a hard - # error ("arguments can only be given if the '--run' option is selected"). - let fileArg = filename.quoteShell - let at = result.find(fileArg) - let switch = "--nimCache:" & nimcache.quoteShell & " " - if at >= 0: result = result[0 ..< at] & switch & result[at .. ^1] - else: result.add " " & switch + switches.add "--nimCache:" & nimcache.quoteShell & " " + # `rfind`, not `find`: the private nimcache path embeds the test's file name + # (`nimcache/tests/destructor/tmove.nim_`), so the FIRST occurrence is + # inside a switch's value. The project file is the last one. + let fileArg = filename.quoteShell + let at = result.rfind(fileArg) + if at >= 0: result = result[0 ..< at] & switches & result[at .. ^1] + else: result.add " " & switches proc icWarmupCache(cmdTemplate, filename, options: string, target: TTarget, extraOptions: string): string = @@ -280,7 +279,8 @@ proc prefillIcCache(warmup, nimcache: string) = if warmup.len == 0 or not dirExists(warmup): return if dirExists(nimcache): return # the test already has its own cache const wanted = [".p.nif", ".p.deps.nif", ".deps.nif", ".s.bif", ".iface.bif", - ".impl.bif", ".edges.bif", ".s.deps.bif", ".t.bif", ".c.nif"] + ".impl.bif", ".edges.bif", ".s.deps.bif", ".t.bif", + ".c.nif", ".cpp.nif"] try: createDir(nimcache) for path in walkFiles(warmup / "*"): @@ -299,7 +299,7 @@ proc prefillIcCache(warmup, nimcache: string) = proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, target: TTarget, extraOptions = ""): TSpec = - if useIc and target == targetC and nimcache.len > 0 and not buildingIcWarmup: + if useIc and target in {targetC, targetCpp} and nimcache.len > 0 and not buildingIcWarmup: prefillIcCache(icWarmupCache(cmdTemplate, filename, options, target, extraOptions), nimcache) result = TSpec(cmd: prepareTestCmd(cmdTemplate, filename, options, nimcache, target,