diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 8bad196801..f6d2094090 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -1324,6 +1324,7 @@ var repDeepCopyTag = registerTag("repdeepcopy") var repEnumToStrTag = registerTag("repenumtostr") var repMethodTag = registerTag("repmethod") var repPureEnumTag = registerTag("reppureenum") +var repCppMemberTag = registerTag("repcppmember") #var repClassTag = registerTag("repclass") var includeTag = registerTag("include") var importTag = registerTag("import") @@ -1401,6 +1402,7 @@ proc registerNifAstTags*() = repEnumToStrTag = registerTag("repenumtostr") repMethodTag = registerTag("repmethod") repPureEnumTag = registerTag("reppureenum") + repCppMemberTag = registerTag("repcppmember") includeTag = registerTag("include") importTag = registerTag("import") implTag = registerTag("implementation") @@ -1719,6 +1721,11 @@ proc writeOp(w: var Writer; content: var IcBuilder; op: LogEntry) = content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() + of CppMemberEntry: + content.addParLe repCppMemberTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) + content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) + content.addParRi() of GenericInstEntry: discard "will only be written later to ensure it is materialized" @@ -3832,6 +3839,7 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag] elif tagIs(cur, "repenumtostr"): loadLogOp(c, result.logOps, cur, EnumToStrEntry, attachedTrace, module) elif tagIs(cur, "repmethod"): loadLogOp(c, result.logOps, cur, MethodEntry, attachedTrace, module) elif tagIs(cur, "reppureenum"): loadLogOp(c, result.logOps, cur, PureEnumEntry, attachedTrace, module) + elif tagIs(cur, "repcppmember"): loadLogOp(c, result.logOps, cur, CppMemberEntry, attachedTrace, module) elif tagIs(cur, "export"): cur.into: while cur.hasMore and cur.kind == DotToken: skip cur # flags / type diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 11f9a4474d..42ccf4c7b0 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -1049,7 +1049,7 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode = type LogEntryKind* = enum HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry, - PureEnumEntry + PureEnumEntry, CppMemberEntry LogEntry* = object kind*: LogEntryKind op*: TTypeAttachedOp diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index a7416e0815..08cab8c13b 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1289,6 +1289,14 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool name = typDesc if isFnConst: fnConst = " const" + if not isCtor: + # The call-site form (`x->salute(@)`), not the mangled Nim name. Set it on + # BOTH paths: whole-program cgen always emitted the out-of-class definition + # (the `else` branch) before any caller, but the per-module backend emits a + # foreign member proc's body in ITS OWN module, so the caller's TU only ever + # reaches the in-class declaration below — and called the member by the + # mangled name (`loo->salute_u0__vireouyks1()`, "struct Loo has no member"). + prc.locImpl.snippet = "$1$2(@)" % [memberOp, name] if isFwdDecl: if isStatic: result.add "static " @@ -1298,9 +1306,7 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool override = " override" superCall = "" else: - if not isCtor: - prc.locImpl.snippet = "$1$2(@)" % [memberOp, name] - elif superCall != "": + if isCtor and superCall != "": superCall = " : " & superCall name = "$1::$2" % [typDesc, name] @@ -1891,11 +1897,30 @@ proc genVTable(result: var Builder, seqs: seq[PSym]) = result.add(cCast(CPointer, seqs[i].loc.snippet)) proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) = + ## The C++/HCR flavour: C++ has no designated initializers, so the RTTI record + ## is a bare variable that the module's `DatInit` fills field by field. cgsym(m, "TNimTypeV2") - m.s[cfsStrData].addDeclWithVisibility(Private): - m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2") if m.config.cmd == cmdNifC: + # Same emit-everywhere split as `genTypeInfoV2Impl`: every `cg` process that + # demands this type declares it `extern`, and the DEFINITION is a droppable + # `'d'` unit the merge stage gives a single owner. Without the split the bare + # `TNimTypeV2 x;` in each TU is a tentative definition — which C's linker + # merges but C++'s does not, so `nim cpp --ic:on` died at link with + # "multiple definition of NTIv2__…". The field ASSIGNMENTS stay in every + # TU's `DatInit`: they are top-level code, not a definition, and every module + # computes the same values. + m.s[cfsStrData].addDeclWithVisibility(Extern): + m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2") + m.s[cfsVars].add(cnifDefDirective(name, "d", icNifName(m, origType))) + var def = newBuilder("") + def.addDeclWithVisibility(Private): + def.addVar(kind = Local, name = name, typ = "TNimTypeV2") + m.s[cfsVars].add extract(def) + m.s[cfsVars].add(cnifEndDefs()) m.icDataDefs.add (name, icNifName(m, origType)) + else: + m.s[cfsStrData].addDeclWithVisibility(Private): + m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2") var flags = 0 if not canFormAcycle(m.g.graph, t): flags = flags or 1 diff --git a/compiler/cgen.nim b/compiler/cgen.nim index cf48f9dc73..4250575c09 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1635,8 +1635,17 @@ proc genProcLvl3*(m: BModule, prc: PSym) = # `extern`/`rtl` pragma at sem time), so its uses are invisible to the # artifact's liveness walk — conservatively keep the definition. defFlags.add 'x' - m.s[cfsProcs].add(cnifDefDirective(stripCnifMarks(prc.loc.snippet), defFlags, - icNifName(m, prc))) + # A C++ member's `loc.snippet` is a CALL PATTERN (`#->salute(@)`), not a + # linker name — and every member of that name, in every class, mints the + # same one. Ownership is assigned per name, so `Loo::salute` and `Foo::salute` + # collided: the merge stage handed both to one artifact and the other TU's + # definition was dropped (undefined vtable at link). Key member definitions by + # their NIF name instead, which is unique by construction. Dots cannot occur + # in a mangled C name, so the two namespaces stay disjoint. + let defName = + if sfCppMember * prc.flags != {}: icNifName(m, prc) + else: stripCnifMarks(prc.loc.snippet) + m.s[cfsProcs].add(cnifDefDirective(defName, defFlags, icNifName(m, prc))) m.s[cfsProcs].add(extract(generatedProc)) m.s[cfsProcs].add(cnifEndDefs()) else: @@ -1661,7 +1670,20 @@ proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} = proc genProcPrototype(m: BModule, sym: PSym) = useHeader(m, sym) - if lfNoDecl in sym.loc.flags or sfCppMember * sym.flags != {}: return + if lfNoDecl in sym.loc.flags: return + if sfCppMember * sym.flags != {}: + # A C++ member is declared INSIDE its class, never as a free prototype — but + # this TU still needs its CALL-SITE name (`x->salute(@)`), and only + # `genMemberProcHeader` derives that (from the pragma's declaration pattern). + # Whole-program cgen got it for free: the module defining the member was code + # generated in the same process, ahead of any caller. The per-module backend + # emits that body in ANOTHER process, so the caller was left with the mangled + # Nim name `fillBackendName` minted and C++ rejected + # `loo->salute_u0__vireouyks1()` ("struct Loo has no member named ..."). + if m.compileToCpp: + var scratch = newBuilder("") + genMemberProcHeader(m, sym, scratch, false, true) + return if lfDynamicLib in sym.loc.flags: if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg": # Under IC per-module cg every demander emits the dynlib proc's DEFINITION @@ -2711,7 +2733,7 @@ proc getCFile*(m: BModule): AbsoluteFile = let ext = if m.compileToCpp: ".nim.cpp" elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m" - else: ".nim.c" + else: icCFileExt(m.config) result = changeFileExt(completeCfilePath(m.config, mangleModuleName(m.config, m.cfilename).AbsoluteFile), ext) when false: diff --git a/compiler/commands.nim b/compiler/commands.nim index 1182a4a377..680ab8427e 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1092,9 +1092,14 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) helpOnError(conf, pass) of "symbolfiles", "incremental", "ic": - if switch.normalize == "symbolfiles": deprecatedAlias(switch, "incremental") + if pass in {passCmd2, passPP} and switch.normalize == "symbolfiles": + deprecatedAlias(switch, "incremental") # xxx maybe also ic, since not in help? - if pass in {passCmd2, passPP}: + # `--ic:on` is read in passCmd1 too: `nim.nim` decides BEFORE config loading + # whether this run is an IC driver (`ensureIcConfig` must produce the + # precompiled config the driver itself then replays), and passCmd1 is the + # only pass that has run by then. + if pass in {passCmd1, passCmd2, passPP}: case arg.normalize of "on": conf.ic = true of "legacy": conf.symbolFiles = v2Sf diff --git a/compiler/deps.nim b/compiler/deps.nim index a812d949b2..9450670edf 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -934,6 +934,19 @@ proc computeForwardedArgs(c: DepContext): seq[string] = # them — phantom outputs that re-fire the build on every rerun). if c.config.selectedGC != gcUnselected: result.add "--mm:" & $c.config.selectedGC + # The children are invoked as `nim m` / `nim nifc`, so the driver's own command + # token (`c`, `cpp`, `ic`) is gone and with it the backend it selected. Name it + # explicitly — `nim cpp --ic:on` must not have its stdlib sem'd and its TUs + # emitted as C. The exception model rides along for the same reason: `nim cpp` + # defaults to `--exceptions:cpp`, which changes both codegen and sem. + if c.config.backend != backendInvalid: + result.add "--backend:" & $c.config.backend + if c.config.exc != excNone: + result.add "--exceptions:" & (case c.config.exc + of excGoto: "goto" + of excCpp: "cpp" + of excQuirky: "quirky" + else: "setjmp") # method dispatch semantics must match across the child processes: # a child compiled without --multimethods:on builds different dispatch # buckets (and rejects calls as ambiguous that multi-dispatch accepts) @@ -1232,7 +1245,7 @@ proc backendCFile(c: DepContext; node: Node): string = if node.id == 0: AbsoluteFile node.files[0].nimFile else: AbsoluteFile node.files[0].modname result = changeFileExt(completeCfilePath(c.config, - mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string + mangleModuleName(c.config, cfilename).AbsoluteFile), icCFileExt(c.config)).string proc computeLiveBackendNodes(c: DepContext): seq[bool] = ## Which nodes the backend must code-generate: the closure reachable from the diff --git a/compiler/icconfig.nim b/compiler/icconfig.nim index a0255c23dc..55c8ad8815 100644 --- a/compiler/icconfig.nim +++ b/compiler/icconfig.nim @@ -32,7 +32,7 @@ ## would misresolve. import options, commands, lineinfos, pathutils, msgs -import std/[algorithm, os, sets, osproc, times, streams, syncio] +import std/[algorithm, os, sets, osproc, times, streams, syncio, strutils] import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse] const @@ -269,11 +269,26 @@ proc ensureIcConfig*(conf: ConfigRef) = # verbatim: all `-`-prefixed switches first (in encounter order), then the # non-switch project token(s). The producer re-reads `nim.cfg` itself. var pargs = @["icconfig", "--icConfigOut:" & outPath] + # The command token is dropped below, so `nim cpp --ic:on` would hand the + # producer a C-backend config: name the backend explicitly. (`nim ic + # --backend:cpp` already carries the switch; the duplicate is harmless.) + if conf.backend != backendInvalid: + pargs.add "--backend:" & $conf.backend var rest: seq[string] = @[] var droppedCmd = false for a in commandLineParams(): if a.len == 0: continue if a[0] == '-': + # `--run`/`-r` must not reach the producer: it only serialises the + # resolved config, has no output binary, and `nim.nim`'s run step asserts + # on the empty `outFile` (`nim cpp --ic:on -r foo.nim`). + var name = "" + var i = 1 + if i < a.len and a[i] == '-': inc i + while i < a.len and a[i] notin {':', '='}: + name.add a[i] + inc i + if normalize(name) in ["r", "run"]: continue pargs.add a elif not droppedCmd: droppedCmd = true # drop the original command token (`ic`/`track`) diff --git a/compiler/main.nim b/compiler/main.nim index 0365eba486..cddf2fc96b 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -29,7 +29,7 @@ when defined(nimPreviewSlimSystem): import ../dist/checksums/src/checksums/sha1 import pipelines -from icconfig import produceIcConfig +from icconfig import produceIcConfig, ensureIcConfig when not defined(nimKochBootstrap): import nifbackend @@ -269,6 +269,28 @@ proc mainCommand*(graph: ModuleGraph) = proc compileToBackend() = customizeForBackend(conf.backend) + if isIcDriver(conf): + # `nim c --ic:on` / `nim cpp --ic:on`: same driver as `nim ic`, entered + # through the ordinary compile command so every backend switch the user + # already knows keeps working (`nim cpp`, `--exceptions:`, `-d:`, ...). + # `customizeForBackend` above has already defined the backend symbol and + # picked the exception model, which is exactly what the per-module + # children must inherit — `computeForwardedArgs` forwards both. + setUseIc(true) + wantMainModule(conf) + setOutFile(conf) + when not defined(nimKochBootstrap): + if conf.icPreparsedConfig.len == 0: + # `--ic:on` came from a `nim.cfg`/`config.nims` rather than the command + # line, so `nim.nim` could not see it before config loading and the + # precompiled config the children replay does not exist yet. Produce it + # now. (The driver then keeps the config IT parsed instead of replaying + # the artifact; both come from the same files.) + ensureIcConfig(conf) + commandIc(conf) + else: + rawMessage(conf, errGenerated, "--ic:on not available in bootstrap build") + return setOutFile(conf) case conf.backend of backendC: commandCompileToC(graph) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index a392c7020f..31ae39828b 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -490,6 +490,49 @@ proc logMethodDef*(g: ModuleGraph; s: PSym) = g.opsLog.add LogEntry(kind: MethodEntry, module: s.itemId.module.int, key: "", sym: s) +proc logCppMember*(g: ModuleGraph; s: PSym) = + ## Log a C++ `{.member.}`/`{.virtual.}`/`{.constructor.}` registration (and the + ## `importcpp` default-initializer flavour) so the NIF backend can rebuild + ## `memberProcsPerType`/`initializersPerType`, which live only in the sem + ## process. Without them the per-module backend emitted the struct WITHOUT its + ## in-class member declarations and the out-of-class definitions did not match + ## ("no declaration matches 'void Doo::memberProc()'"). + ## + ## No type key: `replayCppMember` re-derives the type from the routine's + ## signature exactly as `semCppMember` does, so nothing has to survive the + ## round trip except the routine itself. + if g.config.cmd in {cmdNifC, cmdM}: + g.opsLog.add LogEntry(kind: CppMemberEntry, module: s.itemId.module.int, + key: "", sym: s) + +proc replayCppMember*(g: ModuleGraph; s: PSym) = + ## Inverse of `logCppMember`, mirroring `semstmts.semCppMember`'s derivation. + if s == nil or s.typ == nil: return + if sfImportc notin s.flags: + var typ = if sfConstructor in s.flags: s.typ.returnType else: s.typ.firstParamType + if typ != nil and typ.kind == tyPtr and sfConstructor notin s.flags: + typ = typ.elementType + if typ != nil and typ.kind == tyObject: + let procs = addr g.memberProcsPerType.mgetOrPut(typ.bindingId, @[]) + for prc in procs[]: + if prc == s: return + procs[].add s + else: + let typ = s.typ.returnType + if typ != nil and typ.kind == tyObject and + typ.bindingId notin g.initializersPerType and s.typ.n != nil: + # The default values sem read off the `nkIdentDefs` live on the param syms. + var call = newTree(nkCall, newSymNode(s)) + var isInitializer = s.typ.n.len > 1 + for i in 1 ..< s.typ.n.len: + let p = s.typ.n[i] + if p.kind != nkSym or p.sym.ast == nil or p.sym.ast.kind == nkEmpty: + isInitializer = false + break + call.add p.sym.ast + if isInitializer: + g.initializersPerType[typ.bindingId] = call + proc registerLoadedMethod*(g: ModuleGraph; m: PSym) = ## Rebuild the dispatch buckets from a serialized method registration. ## Buckets group the methods sharing a dispatcher; the dispatcher's BODY @@ -972,6 +1015,8 @@ when not defined(nimKochBootstrap): g.loadedOps[x.op][x.key] = x.sym of EnumToStrEntry: g.loadedEnumToStringProcs[x.key] = x.sym + of CppMemberEntry: + replayCppMember(g, x.sym) of MethodEntry: # only `methodDef` registrations (empty key) rebuild dispatch # buckets; the `addMethodToGeneric` flavor (typeKey key) announces @@ -1162,7 +1207,7 @@ when not defined(nimKochBootstrap): discard "dispatch buckets already rebuilt by registerLoadedHooks" of GenericInstEntry: raiseAssert "GenericInstEntry should not be in the NIF index" - of HookEntry, EnumToStrEntry: + of HookEntry, EnumToStrEntry, CppMemberEntry: discard "already done by registerLoadedHooks" # Register methods per type from NIF index discard "todo" diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index f4b1ce2e20..91ec2774e6 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -750,7 +750,7 @@ proc generateMergeStage(g: ModuleGraph) = let p = line.strip() if p.len > 0: files.add p else: - for artifact in walkFiles(nimcache / "*.c.nif"): + for artifact in walkFiles(nimcache / ("*" & icCFileExt(g.config) & ".nif")): files.add artifact sort files let decision = computeMergeDecision(files) @@ -790,7 +790,7 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) = if targetIsMain: AbsoluteFile toFullPath(g.config, mainFileIdx) else: AbsoluteFile g.config.icBackendModule let cfile = changeFileExt(completeCfilePath(g.config, - mangleModuleName(g.config, cfilename).AbsoluteFile), ".nim.c").string + mangleModuleName(g.config, cfilename).AbsoluteFile), icCFileExt(g.config)).string let artifact = cfile & ".nif" if not fileExists(artifact): rawMessage(g.config, errGenerated, @@ -873,7 +873,7 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) = if not decision.broken: var liveOwners = initHashSet[string]() for cname, owner in decision.owners: - if owner.endsWith(".c.nif") and cname in decision.live: + if owner.endsWith(icCFileExt(g.config) & ".nif") and cname in decision.live: liveOwners.incl owner for owner in liveOwners: let cbase = owner[0 ..< owner.len - ".nif".len] # "@m….nim.c.nif" -> ".c" diff --git a/compiler/nim.nim b/compiler/nim.nim index 789657e216..68765365a2 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -120,7 +120,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = # so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the # driver runs on the exact same config its children will. See icconfig.nim. when not defined(nimKochBootstrap): - if conf.cmd in {cmdIc, cmdTrack}: + if conf.cmd in {cmdIc, cmdTrack} or isIcDriver(conf): ensureIcConfig(conf) var graph = newModuleGraph(cache, conf) diff --git a/compiler/options.nim b/compiler/options.nim index 0b8c06fbe8..a8a874a7f5 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -29,7 +29,7 @@ const nimEnableCovariance* = defined(nimEnableCovariance) - icFormatVersion* = "36" + icFormatVersion* = "37" ## Version of the IC cache format (the sem-NIF module layout written by ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever ## that layout changes: `commandIc` wipes a nimcache whose `ic.version` @@ -940,6 +940,24 @@ proc getOsCacheDir(): string = else: result = getHomeDir() / genSubDir.string +proc isIcDriver*(conf: ConfigRef): bool = + ## True for `nim c --ic:on` / `nim cpp --ic:on`: this process is the `nim ic` + ## DRIVER (it builds the nifmake graph and spawns the per-module children), + ## not a compilation. `nim ic` itself keeps its own `cmdIc` branch. + conf.ic and conf.cmd in {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC} + +proc icCFileExt*(conf: ConfigRef): string = + ## The extension the per-module backend gives a module's translation unit. + ## Mirrors `cgen.getCFile` at BACKEND granularity, which is all the `nim ic` + ## driver can know: it DECLARES every module's `.c`/`.cpp` output to nifmake + ## without loading a single module, so a per-module `{.compile: cpp.}` + ## (`sfCompileToCpp`) is out of reach — and `nim cpp` selects the backend for + ## the whole program anyway. + case conf.backend + of backendCpp: ".nim.cpp" + of backendObjc: ".nim.m" + else: ".nim.c" + proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir = proc nimcacheSuffix(conf: ConfigRef): string = if conf.ideActive: "_nimsuggest" # dedicated cache, never shared with `nim c` diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 9b9e8a75b9..6f9696cea6 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2410,6 +2410,7 @@ proc semCppMember(c: PContext; s: PSym; n: PNode) = localError(c.config, n.info, pragmaName & " must be either ptr to object or object type.") if sameOwners(typ.owner, s.owner) and sameOwners(c.module, s.owner): c.graph.memberProcsPerType.mgetOrPut(typ.bindingId, @[]).add s + logCppMember(c.graph, s) else: localError(c.config, n.info, pragmaName & " procs must be defined in the same scope as the type they are virtual for and it must be a top level scope") @@ -2432,6 +2433,7 @@ proc semCppMember(c: PContext; s: PSym; n: PNode) = inc j if isInitializer: c.graph.initializersPerType[typ.bindingId] = initializerCall + logCppMember(c.graph, s) proc semMethodPrototype(c: PContext; s: PSym; n: PNode) = if s.isGenericRoutine: diff --git a/doc/ic.md b/doc/ic.md index 162b7bbd3e..a6e6860597 100644 --- a/doc/ic.md +++ b/doc/ic.md @@ -2,12 +2,23 @@ Incremental Compilation (IC) ====================================== -The ``nim ic`` command provides incremental compilation for Nim projects. It -decomposes compilation into per-module steps whose results are cached as NIF -files, and uses the external ``nifmake`` build tool to re-run only the steps -whose inputs changed. +``--ic:on`` turns an ordinary compile into an incremental one. It decomposes +compilation into per-module steps whose results are cached as NIF files, and +uses the external ``nifmake`` build tool to re-run only the steps whose inputs +changed. -This document describes **how `nim ic` works today**, including the edge cases +.. code-block:: cmd + + nim c --ic:on myproject.nim + nim cpp --ic:on myproject.nim + +It is a switch on the normal compile commands, not a command of its own, so +everything else keeps working unchanged: ``cpp`` and ``objc`` backends, ``-r``, +``-d:release``, ``--exceptions:``, and a project-wide opt-in from ``nim.cfg`` / +``config.nims``. The older spelling ``nim ic`` still works and drives the same +code, but it is the C backend only and cannot run the binary it built. + +This document describes **how IC works today**, including the edge cases that shaped the current design. The per-module backend rewrite that earlier editions of this document listed as a *Plan* has **landed**: the whole-program, reuse/redirect/def-retention backend is gone and codegen is now a set of @@ -16,7 +27,7 @@ reuse/redirect/def-retention backend is gone and codegen is now a set of Overview ======== -The pipeline has two halves driven by one process (`nim ic`, `commandIc` in +The pipeline has two halves driven by one process (the *driver*, `commandIc` in ``compiler/deps.nim``) that constructs a dependency graph, writes a build file, and hands it to ``nifmake``: @@ -220,7 +231,7 @@ Edge cases (and why the machinery exists) - **Config cost.** Each child re-parsing `nim.cfg` + re-running `config.nims` in the VM was ~80 ms; replaced by a precompiled `ic_config.cfg.nif` replayed in `loadConfigs` (`compiler/icconfig.nim`). -- **`koch bootic`** bootstraps the compiler through `nim ic` (a 3-iteration +- **`koch bootic`** bootstraps the compiler through `--ic:on` (a 3-iteration fixed-point check). It writes its binary to ``bin/nim_ic`` and never clobbers ``bin/nim``. @@ -247,7 +258,7 @@ Known residual hack Status and performance ====================== -`nim ic` self-builds the compiler (`koch bootic`'s byte-identical fixed-point +IC self-builds the compiler (`koch bootic`'s byte-identical fixed-point check) under both `orc` and `--mm:refc`, and passes the external-package CI set. Cold full bootstrap on a 32-core box (`-d:release`, **no edits** — IC's worst @@ -256,7 +267,7 @@ case, since incremental reuse is not exercised): | | wall | notes | | - | ---- | ----- | | `koch boot` (classic) | ~1m00s | reference | -| `koch bootic` (`nim ic`) | ~1m39s | **~1.66×** | +| `koch bootic` (`--ic:on`) | ~1m39s | **~1.66×** | This is down from ~7.5× in the whole-program-backend era. IC does modestly more aggregate work (more processes, NIF re-parsing of imports per process), but on a @@ -412,7 +423,7 @@ Testing IC Two mechanisms, at very different scales. **`tests/ic` — metamorphic tests.** A `t*.nim` whose body contains `#? metamorphic` -drives a sequence of cross-module edits through `nim ic` in one fixed build +drives a sequence of cross-module edits through the IC driver in one fixed build directory (see `testament/categories.nim`, `runMetamorphicIcTest`). Directives: | directive | effect | @@ -436,11 +447,13 @@ had nowhere to live on a serialized sym node, so every first assignment to a destructor-bearing local became `=sink` over zeroed memory). `koch bootic` has the same blind spot — it proves the compiler reproduces *itself*. -**`testament --ic` — the whole corpus.** Compiles every C-target test with -`nim ic` instead of `nim c`, so IC inherits the existing ~10k programs and their -expected output instead of the handful written for it by hand. Tests that -override the command (`cmd: "nim c --gc:arc $file"`) are rewritten too, and get a -private nimcache; without one they would share a cache and thrash it. +**`testament --ic` — the whole corpus.** Appends `--ic:on` to every C and C++ +test compile, so IC inherits the existing ~10k programs and their expected +output instead of the handful written for it by hand. Because it is a switch and +not a command, a test that overrides the command wholesale (`cmd: "nim cpp -r +$file"`) simply gains the switch — no verb rewriting, and the C++ corpus comes +along for free. Each also gets a private nimcache; without one they would share +a cache and thrash it. To keep that affordable, testament borrows nimony's hastur model (`warmupSharedCache` + `prefillFromWarmup`): a generated warmup program pulling in @@ -458,7 +471,7 @@ Measured on `tests/destructor` (97 test runs, 32-core box): | | cold | warm | | - | ---- | ---- | | `nim c` | 35s | 32s | -| `nim ic` | ~3m30 | **9.8s** | +| `--ic:on` | ~3m30 | **9.8s** | The warm number is the developer loop and it is 3.2x faster than the classic backend; the cold number is paid once per configuration and then cached on disk. @@ -471,3 +484,33 @@ hint, a warning — all of it is produced by the process that actually runs, so build that reuses every artifact prints nothing. Tests that check `nimout` (and anything you are debugging by eye) therefore need a cold cache; running the same test twice in a row makes the second run's `nimout` empty. + +The C++ backend +=============== + +``nim cpp --ic:on`` works, and `tests/cpp` passes under it. Three things had to +change for that, and they are worth knowing because they are the shape of every +"C++ needs the whole program" problem the per-module backend has: + +* **The driver must name the right file.** ``deps.nim`` DECLARES each module's + translation unit to ``nifmake`` without loading a single module, so it cannot + ask ``cgen.getCFile``; ``options.icCFileExt`` mirrors that formula at backend + granularity (``.nim.cpp`` / ``.nim.m`` / ``.nim.c``). + +* **C++ has no designated initializers**, so the RTTI record is a bare variable + that ``DatInit`` fills field by field. That bare ``TNimTypeV2 x;`` is a + tentative definition, which C's linker merges and C++'s does not — every TU + that demanded the type defined it. It now gets the same extern-declaration + + owned-``'d'``-definition split the C flavour has. + +* **A C++ member is declared inside its class.** ``memberProcsPerType`` and + ``initializersPerType`` live only in the sem process, so the backend emitted + the struct WITHOUT its member declarations; they are replayed from a + ``(repcppmember …)`` log entry now (``modulegraphs.replayCppMember`` re-derives + the type from the routine's signature, exactly as ``semCppMember`` does). + Two follow-on details: a member's ``loc.snippet`` is a CALL PATTERN + (``#->salute(@)``), so it must be computed even in the TU that only *calls* the + member (whole-program cgen got that for free by generating the defining module + first), and it is not a linker name — every ``salute`` member in every class + mints the same one, so definitions are keyed by their NIF name in the merge + stage instead. diff --git a/koch.nim b/koch.nim index 50d76e03d1..225956c639 100644 --- a/koch.nim +++ b/koch.nim @@ -76,7 +76,7 @@ Options: --skipIntegrityCheck skips integrity check when booting the compiler Possible Commands: boot [options] bootstraps with given command line options - bootic [options] bootstraps via the incremental compiler (`nim ic`) + bootic [options] bootstraps via the incremental compiler (`--ic:on`) distrohelper [bindir] helper for distro packagers tools builds Nim related tools toolsNoExternal builds Nim related tools (except external tools, @@ -450,7 +450,7 @@ proc bootic(args: string, skipIntegrityCheck: bool) = # everything. if i > 0: removeDir smartNimcache let nimi = if i == 0: nimStart else: i.thVersion - exec "$# ic --nimcache:$# $# compiler" / "nim.nim" % + exec "$# c --ic:on --nimcache:$# $# compiler" / "nim.nim" % [nimi, smartNimcache, args] if sameFileContent(output, i.thVersion): copyExe(output, finalDest) @@ -615,7 +615,7 @@ proc runIcTestFile(inp: string) = for fragment in content.split("#!EDIT!#"): let file = inp.replace(".nim", "_temp.nim") writeFile(file, fragment) - var cmd = nimExe & " ic --hint:Conf:off --warnings:off " + var cmd = nimExe & " c --ic:on --hint:Conf:off --warnings:off " cmd.add quoteShell(file) exec(cmd) diff --git a/testament/testament.nim b/testament/testament.nim index 478184fe28..bf2bb87c49 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -201,32 +201,31 @@ proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, var options = target.defaultOptions & ' ' & options if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) options.add ' ' & extraOptions - # `--ic` swaps the C target's command; every other target is left alone (the - # incremental compiler has a C backend only). - let targetCmd = - if useIc and target == targetC: "ic" else: targetToCmd[target] # we avoid using `parseCmdLine` which is buggy, refs bug #14343 - result = cmdTemplate % ["target", targetCmd, + result = cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, "filedir", filename.getFileDir(), "nim", compilerPrefix] - if useIc and target == targetC: - # Roughly half the corpus overrides the command wholesale (`cmd: "nim c - # --gc:arc $file"`), which neither goes through `$target` nor picks up - # `$options` — so those tests would silently keep using the classic backend - # and share one nimcache. Rewrite the compile verb and give them a private - # cache, which is what makes them incremental at all. - let prefix = compilerPrefix & " c " - if result.startsWith(prefix): - result = compilerPrefix & " ic " & result[prefix.len .. ^1] + if useIc and target in {targetC, targetCpp}: + # `--ic:on` turns the ordinary compile command into the IC driver, so the + # verb is left alone: roughly half the corpus overrides the command wholesale + # (`cmd: "nim c --gc:arc $file"`), which neither goes through `$target` nor + # picks up `$options`, and such a test now simply gains the switch. Each also + # gets a private nimcache, which is what makes it incremental at all. + # + # Switches must land BEFORE the project file: anything after it is swallowed + # into `config.arguments`, and a non-empty `arguments` without `--run` is a + # hard error ("arguments can only be given if the '--run' option is + # selected"). + var switches = "--ic:on " if nimcache.len > 0 and "--nimCache:" notin result and "--nimcache:" notin result: - # Must land BEFORE the project file: anything after it is swallowed into - # `config.arguments`, and a non-empty `arguments` without `--run` is a hard - # error ("arguments can only be given if the '--run' option is selected"). - let fileArg = filename.quoteShell - let at = result.find(fileArg) - let switch = "--nimCache:" & nimcache.quoteShell & " " - if at >= 0: result = result[0 ..< at] & switch & result[at .. ^1] - else: result.add " " & switch + switches.add "--nimCache:" & nimcache.quoteShell & " " + # `rfind`, not `find`: the private nimcache path embeds the test's file name + # (`nimcache/tests/destructor/tmove.nim_`), so the FIRST occurrence is + # inside a switch's value. The project file is the last one. + let fileArg = filename.quoteShell + let at = result.rfind(fileArg) + if at >= 0: result = result[0 ..< at] & switches & result[at .. ^1] + else: result.add " " & switches proc icWarmupCache(cmdTemplate, filename, options: string, target: TTarget, extraOptions: string): string = @@ -280,7 +279,8 @@ proc prefillIcCache(warmup, nimcache: string) = if warmup.len == 0 or not dirExists(warmup): return if dirExists(nimcache): return # the test already has its own cache const wanted = [".p.nif", ".p.deps.nif", ".deps.nif", ".s.bif", ".iface.bif", - ".impl.bif", ".edges.bif", ".s.deps.bif", ".t.bif", ".c.nif"] + ".impl.bif", ".edges.bif", ".s.deps.bif", ".t.bif", + ".c.nif", ".cpp.nif"] try: createDir(nimcache) for path in walkFiles(warmup / "*"): @@ -299,7 +299,7 @@ proc prefillIcCache(warmup, nimcache: string) = proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, target: TTarget, extraOptions = ""): TSpec = - if useIc and target == targetC and nimcache.len > 0 and not buildingIcWarmup: + if useIc and target in {targetC, targetCpp} and nimcache.len > 0 and not buildingIcWarmup: prefillIcCache(icWarmupCache(cmdTemplate, filename, options, target, extraOptions), nimcache) result = TSpec(cmd: prepareTestCmd(cmdTemplate, filename, options, nimcache, target,