mirror of
https://github.com/nim-lang/Nim.git
synced 2026-09-01 19:33:42 +00:00
IC: nim c --ic:on / nim cpp --ic:on replace the nim ic command
`nim ic` was a command of its own, which made it the C backend only and cut it
off from everything the ordinary compile commands accept. `--ic:on` is a switch
on `nim c` / `nim cpp` / `nim objc` instead, so `-r`, `-d:release`,
`--exceptions:`, and a project-wide opt-in from `nim.cfg` / `config.nims` all
work. `nim ic` still resolves to the same driver; `koch bootic`, `koch ic` and
`testament --ic` now go through the switch.
The switch was already parsed into a `conf.ic` nobody read. It is now read in
`passCmd1` as well, because `nim.nim` has to decide whether this run is an IC
DRIVER before config loading (`ensureIcConfig` produces the precompiled config
the driver itself replays); when the switch comes from a config file instead,
`main.nim` produces it late.
**C++.** `tests/cpp` passes under `--ic:on`, matching its classic result. Four
fixes, three of them the shape of every "C++ needs the whole program" problem the
per-module backend has:
* The driver DECLARES each module's translation unit to nifmake without loading a
module, so it cannot ask `cgen.getCFile` — and it hardcoded `.nim.c`, so the
merge stage went looking for `.c.nif` next to the `.cpp.nif` the children had
written. `options.icCFileExt` mirrors the formula at backend granularity.
* C++ has no designated initializers, so the RTTI record is a bare variable that
`DatInit` fills field by field. A bare `TNimTypeV2 x;` is a tentative
definition — C's linker merges those, C++'s does not — so every TU that
demanded the type defined it ("multiple definition of NTIv2__…"). It now gets
the same extern-declaration + owned-`'d'`-definition split the C flavour has.
* `memberProcsPerType` / `initializersPerType` live only in the sem process, so
the backend emitted a struct WITHOUT its in-class member declarations and the
out-of-class definitions did not match ("no declaration matches
'void Doo::memberProc()'"). They are replayed from a new `(repcppmember …)`
log entry; `replayCppMember` re-derives the type from the routine's signature
exactly as `semCppMember` does, so no type key has to survive the round trip.
* Two follow-ons for members: `loc.snippet` is a CALL PATTERN (`#->salute(@)`),
and only `genMemberProcHeader` derives it — whole-program cgen got it for free
by generating the defining module first, but the per-module backend emits that
body in another process, leaving the caller with the mangled Nim name
(`loo->salute_u0__vireouyks1()`). And that pattern is not a linker name: every
`salute` member in every class mints the same one, so the merge stage handed
them all to one artifact and dropped the rest (undefined vtable at link).
Member definitions are keyed by their NIF name there instead.
`--run` is now dropped when re-invoking for the config artifact: the producer has
no output binary and `nim.nim`'s run step asserted on the empty `outFile`.
testament's `--ic` appends the switch rather than rewriting the compile verb, so
a test that overrides `cmd:` wholesale keeps its verb, and the C++ corpus is
covered too (it never was — the old rewrite only matched `nim c `).
`icFormatVersion` 36 -> 37 for the new log entry. `koch bootic` reaches its
byte-identical fixed point through the new entry point; `koch ic` passes;
`tests/ic`, `tests/destructor` (3 known `--newruntime` failures) and the classic
categories are unchanged.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
araq
@@ -1324,6 +1324,7 @@ var repDeepCopyTag = registerTag("repdeepcopy")
|
||||
var repEnumToStrTag = registerTag("repenumtostr")
|
||||
var repMethodTag = registerTag("repmethod")
|
||||
var repPureEnumTag = registerTag("reppureenum")
|
||||
var repCppMemberTag = registerTag("repcppmember")
|
||||
#var repClassTag = registerTag("repclass")
|
||||
var includeTag = registerTag("include")
|
||||
var importTag = registerTag("import")
|
||||
@@ -1401,6 +1402,7 @@ proc registerNifAstTags*() =
|
||||
repEnumToStrTag = registerTag("repenumtostr")
|
||||
repMethodTag = registerTag("repmethod")
|
||||
repPureEnumTag = registerTag("reppureenum")
|
||||
repCppMemberTag = registerTag("repcppmember")
|
||||
includeTag = registerTag("include")
|
||||
importTag = registerTag("import")
|
||||
implTag = registerTag("implementation")
|
||||
@@ -1719,6 +1721,11 @@ proc writeOp(w: var Writer; content: var IcBuilder; op: LogEntry) =
|
||||
content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo)
|
||||
content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo)
|
||||
content.addParRi()
|
||||
of CppMemberEntry:
|
||||
content.addParLe repCppMemberTag, NoLineInfo
|
||||
content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo)
|
||||
content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo)
|
||||
content.addParRi()
|
||||
of GenericInstEntry:
|
||||
discard "will only be written later to ensure it is materialized"
|
||||
|
||||
@@ -3832,6 +3839,7 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]
|
||||
elif tagIs(cur, "repenumtostr"): loadLogOp(c, result.logOps, cur, EnumToStrEntry, attachedTrace, module)
|
||||
elif tagIs(cur, "repmethod"): loadLogOp(c, result.logOps, cur, MethodEntry, attachedTrace, module)
|
||||
elif tagIs(cur, "reppureenum"): loadLogOp(c, result.logOps, cur, PureEnumEntry, attachedTrace, module)
|
||||
elif tagIs(cur, "repcppmember"): loadLogOp(c, result.logOps, cur, CppMemberEntry, attachedTrace, module)
|
||||
elif tagIs(cur, "export"):
|
||||
cur.into:
|
||||
while cur.hasMore and cur.kind == DotToken: skip cur # flags / type
|
||||
|
||||
@@ -1049,7 +1049,7 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode =
|
||||
type
|
||||
LogEntryKind* = enum
|
||||
HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry,
|
||||
PureEnumEntry
|
||||
PureEnumEntry, CppMemberEntry
|
||||
LogEntry* = object
|
||||
kind*: LogEntryKind
|
||||
op*: TTypeAttachedOp
|
||||
|
||||
@@ -1289,6 +1289,14 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool
|
||||
name = typDesc
|
||||
if isFnConst:
|
||||
fnConst = " const"
|
||||
if not isCtor:
|
||||
# The call-site form (`x->salute(@)`), not the mangled Nim name. Set it on
|
||||
# BOTH paths: whole-program cgen always emitted the out-of-class definition
|
||||
# (the `else` branch) before any caller, but the per-module backend emits a
|
||||
# foreign member proc's body in ITS OWN module, so the caller's TU only ever
|
||||
# reaches the in-class declaration below — and called the member by the
|
||||
# mangled name (`loo->salute_u0__vireouyks1()`, "struct Loo has no member").
|
||||
prc.locImpl.snippet = "$1$2(@)" % [memberOp, name]
|
||||
if isFwdDecl:
|
||||
if isStatic:
|
||||
result.add "static "
|
||||
@@ -1298,9 +1306,7 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool
|
||||
override = " override"
|
||||
superCall = ""
|
||||
else:
|
||||
if not isCtor:
|
||||
prc.locImpl.snippet = "$1$2(@)" % [memberOp, name]
|
||||
elif superCall != "":
|
||||
if isCtor and superCall != "":
|
||||
superCall = " : " & superCall
|
||||
|
||||
name = "$1::$2" % [typDesc, name]
|
||||
@@ -1891,11 +1897,30 @@ proc genVTable(result: var Builder, seqs: seq[PSym]) =
|
||||
result.add(cCast(CPointer, seqs[i].loc.snippet))
|
||||
|
||||
proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) =
|
||||
## The C++/HCR flavour: C++ has no designated initializers, so the RTTI record
|
||||
## is a bare variable that the module's `DatInit` fills field by field.
|
||||
cgsym(m, "TNimTypeV2")
|
||||
m.s[cfsStrData].addDeclWithVisibility(Private):
|
||||
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
|
||||
if m.config.cmd == cmdNifC:
|
||||
# Same emit-everywhere split as `genTypeInfoV2Impl`: every `cg` process that
|
||||
# demands this type declares it `extern`, and the DEFINITION is a droppable
|
||||
# `'d'` unit the merge stage gives a single owner. Without the split the bare
|
||||
# `TNimTypeV2 x;` in each TU is a tentative definition — which C's linker
|
||||
# merges but C++'s does not, so `nim cpp --ic:on` died at link with
|
||||
# "multiple definition of NTIv2__…". The field ASSIGNMENTS stay in every
|
||||
# TU's `DatInit`: they are top-level code, not a definition, and every module
|
||||
# computes the same values.
|
||||
m.s[cfsStrData].addDeclWithVisibility(Extern):
|
||||
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
|
||||
m.s[cfsVars].add(cnifDefDirective(name, "d", icNifName(m, origType)))
|
||||
var def = newBuilder("")
|
||||
def.addDeclWithVisibility(Private):
|
||||
def.addVar(kind = Local, name = name, typ = "TNimTypeV2")
|
||||
m.s[cfsVars].add extract(def)
|
||||
m.s[cfsVars].add(cnifEndDefs())
|
||||
m.icDataDefs.add (name, icNifName(m, origType))
|
||||
else:
|
||||
m.s[cfsStrData].addDeclWithVisibility(Private):
|
||||
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
|
||||
|
||||
var flags = 0
|
||||
if not canFormAcycle(m.g.graph, t): flags = flags or 1
|
||||
|
||||
@@ -1635,8 +1635,17 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
|
||||
# `extern`/`rtl` pragma at sem time), so its uses are invisible to the
|
||||
# artifact's liveness walk — conservatively keep the definition.
|
||||
defFlags.add 'x'
|
||||
m.s[cfsProcs].add(cnifDefDirective(stripCnifMarks(prc.loc.snippet), defFlags,
|
||||
icNifName(m, prc)))
|
||||
# A C++ member's `loc.snippet` is a CALL PATTERN (`#->salute(@)`), not a
|
||||
# linker name — and every member of that name, in every class, mints the
|
||||
# same one. Ownership is assigned per name, so `Loo::salute` and `Foo::salute`
|
||||
# collided: the merge stage handed both to one artifact and the other TU's
|
||||
# definition was dropped (undefined vtable at link). Key member definitions by
|
||||
# their NIF name instead, which is unique by construction. Dots cannot occur
|
||||
# in a mangled C name, so the two namespaces stay disjoint.
|
||||
let defName =
|
||||
if sfCppMember * prc.flags != {}: icNifName(m, prc)
|
||||
else: stripCnifMarks(prc.loc.snippet)
|
||||
m.s[cfsProcs].add(cnifDefDirective(defName, defFlags, icNifName(m, prc)))
|
||||
m.s[cfsProcs].add(extract(generatedProc))
|
||||
m.s[cfsProcs].add(cnifEndDefs())
|
||||
else:
|
||||
@@ -1661,7 +1670,20 @@ proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} =
|
||||
|
||||
proc genProcPrototype(m: BModule, sym: PSym) =
|
||||
useHeader(m, sym)
|
||||
if lfNoDecl in sym.loc.flags or sfCppMember * sym.flags != {}: return
|
||||
if lfNoDecl in sym.loc.flags: return
|
||||
if sfCppMember * sym.flags != {}:
|
||||
# A C++ member is declared INSIDE its class, never as a free prototype — but
|
||||
# this TU still needs its CALL-SITE name (`x->salute(@)`), and only
|
||||
# `genMemberProcHeader` derives that (from the pragma's declaration pattern).
|
||||
# Whole-program cgen got it for free: the module defining the member was code
|
||||
# generated in the same process, ahead of any caller. The per-module backend
|
||||
# emits that body in ANOTHER process, so the caller was left with the mangled
|
||||
# Nim name `fillBackendName` minted and C++ rejected
|
||||
# `loo->salute_u0__vireouyks1()` ("struct Loo has no member named ...").
|
||||
if m.compileToCpp:
|
||||
var scratch = newBuilder("")
|
||||
genMemberProcHeader(m, sym, scratch, false, true)
|
||||
return
|
||||
if lfDynamicLib in sym.loc.flags:
|
||||
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg":
|
||||
# Under IC per-module cg every demander emits the dynlib proc's DEFINITION
|
||||
@@ -2711,7 +2733,7 @@ proc getCFile*(m: BModule): AbsoluteFile =
|
||||
let ext =
|
||||
if m.compileToCpp: ".nim.cpp"
|
||||
elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m"
|
||||
else: ".nim.c"
|
||||
else: icCFileExt(m.config)
|
||||
result = changeFileExt(completeCfilePath(m.config, mangleModuleName(m.config, m.cfilename).AbsoluteFile), ext)
|
||||
|
||||
when false:
|
||||
|
||||
@@ -1092,9 +1092,14 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
|
||||
expectNoArg(conf, switch, arg, pass, info)
|
||||
helpOnError(conf, pass)
|
||||
of "symbolfiles", "incremental", "ic":
|
||||
if switch.normalize == "symbolfiles": deprecatedAlias(switch, "incremental")
|
||||
if pass in {passCmd2, passPP} and switch.normalize == "symbolfiles":
|
||||
deprecatedAlias(switch, "incremental")
|
||||
# xxx maybe also ic, since not in help?
|
||||
if pass in {passCmd2, passPP}:
|
||||
# `--ic:on` is read in passCmd1 too: `nim.nim` decides BEFORE config loading
|
||||
# whether this run is an IC driver (`ensureIcConfig` must produce the
|
||||
# precompiled config the driver itself then replays), and passCmd1 is the
|
||||
# only pass that has run by then.
|
||||
if pass in {passCmd1, passCmd2, passPP}:
|
||||
case arg.normalize
|
||||
of "on": conf.ic = true
|
||||
of "legacy": conf.symbolFiles = v2Sf
|
||||
|
||||
@@ -934,6 +934,19 @@ proc computeForwardedArgs(c: DepContext): seq[string] =
|
||||
# them — phantom outputs that re-fire the build on every rerun).
|
||||
if c.config.selectedGC != gcUnselected:
|
||||
result.add "--mm:" & $c.config.selectedGC
|
||||
# The children are invoked as `nim m` / `nim nifc`, so the driver's own command
|
||||
# token (`c`, `cpp`, `ic`) is gone and with it the backend it selected. Name it
|
||||
# explicitly — `nim cpp --ic:on` must not have its stdlib sem'd and its TUs
|
||||
# emitted as C. The exception model rides along for the same reason: `nim cpp`
|
||||
# defaults to `--exceptions:cpp`, which changes both codegen and sem.
|
||||
if c.config.backend != backendInvalid:
|
||||
result.add "--backend:" & $c.config.backend
|
||||
if c.config.exc != excNone:
|
||||
result.add "--exceptions:" & (case c.config.exc
|
||||
of excGoto: "goto"
|
||||
of excCpp: "cpp"
|
||||
of excQuirky: "quirky"
|
||||
else: "setjmp")
|
||||
# method dispatch semantics must match across the child processes:
|
||||
# a child compiled without --multimethods:on builds different dispatch
|
||||
# buckets (and rejects calls as ambiguous that multi-dispatch accepts)
|
||||
@@ -1232,7 +1245,7 @@ proc backendCFile(c: DepContext; node: Node): string =
|
||||
if node.id == 0: AbsoluteFile node.files[0].nimFile
|
||||
else: AbsoluteFile node.files[0].modname
|
||||
result = changeFileExt(completeCfilePath(c.config,
|
||||
mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string
|
||||
mangleModuleName(c.config, cfilename).AbsoluteFile), icCFileExt(c.config)).string
|
||||
|
||||
proc computeLiveBackendNodes(c: DepContext): seq[bool] =
|
||||
## Which nodes the backend must code-generate: the closure reachable from the
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
## would misresolve.
|
||||
|
||||
import options, commands, lineinfos, pathutils, msgs
|
||||
import std/[algorithm, os, sets, osproc, times, streams, syncio]
|
||||
import std/[algorithm, os, sets, osproc, times, streams, syncio, strutils]
|
||||
import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse]
|
||||
|
||||
const
|
||||
@@ -269,11 +269,26 @@ proc ensureIcConfig*(conf: ConfigRef) =
|
||||
# verbatim: all `-`-prefixed switches first (in encounter order), then the
|
||||
# non-switch project token(s). The producer re-reads `nim.cfg` itself.
|
||||
var pargs = @["icconfig", "--icConfigOut:" & outPath]
|
||||
# The command token is dropped below, so `nim cpp --ic:on` would hand the
|
||||
# producer a C-backend config: name the backend explicitly. (`nim ic
|
||||
# --backend:cpp` already carries the switch; the duplicate is harmless.)
|
||||
if conf.backend != backendInvalid:
|
||||
pargs.add "--backend:" & $conf.backend
|
||||
var rest: seq[string] = @[]
|
||||
var droppedCmd = false
|
||||
for a in commandLineParams():
|
||||
if a.len == 0: continue
|
||||
if a[0] == '-':
|
||||
# `--run`/`-r` must not reach the producer: it only serialises the
|
||||
# resolved config, has no output binary, and `nim.nim`'s run step asserts
|
||||
# on the empty `outFile` (`nim cpp --ic:on -r foo.nim`).
|
||||
var name = ""
|
||||
var i = 1
|
||||
if i < a.len and a[i] == '-': inc i
|
||||
while i < a.len and a[i] notin {':', '='}:
|
||||
name.add a[i]
|
||||
inc i
|
||||
if normalize(name) in ["r", "run"]: continue
|
||||
pargs.add a
|
||||
elif not droppedCmd:
|
||||
droppedCmd = true # drop the original command token (`ic`/`track`)
|
||||
|
||||
@@ -29,7 +29,7 @@ when defined(nimPreviewSlimSystem):
|
||||
import ../dist/checksums/src/checksums/sha1
|
||||
|
||||
import pipelines
|
||||
from icconfig import produceIcConfig
|
||||
from icconfig import produceIcConfig, ensureIcConfig
|
||||
|
||||
when not defined(nimKochBootstrap):
|
||||
import nifbackend
|
||||
@@ -269,6 +269,28 @@ proc mainCommand*(graph: ModuleGraph) =
|
||||
|
||||
proc compileToBackend() =
|
||||
customizeForBackend(conf.backend)
|
||||
if isIcDriver(conf):
|
||||
# `nim c --ic:on` / `nim cpp --ic:on`: same driver as `nim ic`, entered
|
||||
# through the ordinary compile command so every backend switch the user
|
||||
# already knows keeps working (`nim cpp`, `--exceptions:`, `-d:`, ...).
|
||||
# `customizeForBackend` above has already defined the backend symbol and
|
||||
# picked the exception model, which is exactly what the per-module
|
||||
# children must inherit — `computeForwardedArgs` forwards both.
|
||||
setUseIc(true)
|
||||
wantMainModule(conf)
|
||||
setOutFile(conf)
|
||||
when not defined(nimKochBootstrap):
|
||||
if conf.icPreparsedConfig.len == 0:
|
||||
# `--ic:on` came from a `nim.cfg`/`config.nims` rather than the command
|
||||
# line, so `nim.nim` could not see it before config loading and the
|
||||
# precompiled config the children replay does not exist yet. Produce it
|
||||
# now. (The driver then keeps the config IT parsed instead of replaying
|
||||
# the artifact; both come from the same files.)
|
||||
ensureIcConfig(conf)
|
||||
commandIc(conf)
|
||||
else:
|
||||
rawMessage(conf, errGenerated, "--ic:on not available in bootstrap build")
|
||||
return
|
||||
setOutFile(conf)
|
||||
case conf.backend
|
||||
of backendC: commandCompileToC(graph)
|
||||
|
||||
@@ -490,6 +490,49 @@ proc logMethodDef*(g: ModuleGraph; s: PSym) =
|
||||
g.opsLog.add LogEntry(kind: MethodEntry, module: s.itemId.module.int,
|
||||
key: "", sym: s)
|
||||
|
||||
proc logCppMember*(g: ModuleGraph; s: PSym) =
|
||||
## Log a C++ `{.member.}`/`{.virtual.}`/`{.constructor.}` registration (and the
|
||||
## `importcpp` default-initializer flavour) so the NIF backend can rebuild
|
||||
## `memberProcsPerType`/`initializersPerType`, which live only in the sem
|
||||
## process. Without them the per-module backend emitted the struct WITHOUT its
|
||||
## in-class member declarations and the out-of-class definitions did not match
|
||||
## ("no declaration matches 'void Doo::memberProc()'").
|
||||
##
|
||||
## No type key: `replayCppMember` re-derives the type from the routine's
|
||||
## signature exactly as `semCppMember` does, so nothing has to survive the
|
||||
## round trip except the routine itself.
|
||||
if g.config.cmd in {cmdNifC, cmdM}:
|
||||
g.opsLog.add LogEntry(kind: CppMemberEntry, module: s.itemId.module.int,
|
||||
key: "", sym: s)
|
||||
|
||||
proc replayCppMember*(g: ModuleGraph; s: PSym) =
|
||||
## Inverse of `logCppMember`, mirroring `semstmts.semCppMember`'s derivation.
|
||||
if s == nil or s.typ == nil: return
|
||||
if sfImportc notin s.flags:
|
||||
var typ = if sfConstructor in s.flags: s.typ.returnType else: s.typ.firstParamType
|
||||
if typ != nil and typ.kind == tyPtr and sfConstructor notin s.flags:
|
||||
typ = typ.elementType
|
||||
if typ != nil and typ.kind == tyObject:
|
||||
let procs = addr g.memberProcsPerType.mgetOrPut(typ.bindingId, @[])
|
||||
for prc in procs[]:
|
||||
if prc == s: return
|
||||
procs[].add s
|
||||
else:
|
||||
let typ = s.typ.returnType
|
||||
if typ != nil and typ.kind == tyObject and
|
||||
typ.bindingId notin g.initializersPerType and s.typ.n != nil:
|
||||
# The default values sem read off the `nkIdentDefs` live on the param syms.
|
||||
var call = newTree(nkCall, newSymNode(s))
|
||||
var isInitializer = s.typ.n.len > 1
|
||||
for i in 1 ..< s.typ.n.len:
|
||||
let p = s.typ.n[i]
|
||||
if p.kind != nkSym or p.sym.ast == nil or p.sym.ast.kind == nkEmpty:
|
||||
isInitializer = false
|
||||
break
|
||||
call.add p.sym.ast
|
||||
if isInitializer:
|
||||
g.initializersPerType[typ.bindingId] = call
|
||||
|
||||
proc registerLoadedMethod*(g: ModuleGraph; m: PSym) =
|
||||
## Rebuild the dispatch buckets from a serialized method registration.
|
||||
## Buckets group the methods sharing a dispatcher; the dispatcher's BODY
|
||||
@@ -972,6 +1015,8 @@ when not defined(nimKochBootstrap):
|
||||
g.loadedOps[x.op][x.key] = x.sym
|
||||
of EnumToStrEntry:
|
||||
g.loadedEnumToStringProcs[x.key] = x.sym
|
||||
of CppMemberEntry:
|
||||
replayCppMember(g, x.sym)
|
||||
of MethodEntry:
|
||||
# only `methodDef` registrations (empty key) rebuild dispatch
|
||||
# buckets; the `addMethodToGeneric` flavor (typeKey key) announces
|
||||
@@ -1162,7 +1207,7 @@ when not defined(nimKochBootstrap):
|
||||
discard "dispatch buckets already rebuilt by registerLoadedHooks"
|
||||
of GenericInstEntry:
|
||||
raiseAssert "GenericInstEntry should not be in the NIF index"
|
||||
of HookEntry, EnumToStrEntry:
|
||||
of HookEntry, EnumToStrEntry, CppMemberEntry:
|
||||
discard "already done by registerLoadedHooks"
|
||||
# Register methods per type from NIF index
|
||||
discard "todo"
|
||||
|
||||
@@ -750,7 +750,7 @@ proc generateMergeStage(g: ModuleGraph) =
|
||||
let p = line.strip()
|
||||
if p.len > 0: files.add p
|
||||
else:
|
||||
for artifact in walkFiles(nimcache / "*.c.nif"):
|
||||
for artifact in walkFiles(nimcache / ("*" & icCFileExt(g.config) & ".nif")):
|
||||
files.add artifact
|
||||
sort files
|
||||
let decision = computeMergeDecision(files)
|
||||
@@ -790,7 +790,7 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
|
||||
if targetIsMain: AbsoluteFile toFullPath(g.config, mainFileIdx)
|
||||
else: AbsoluteFile g.config.icBackendModule
|
||||
let cfile = changeFileExt(completeCfilePath(g.config,
|
||||
mangleModuleName(g.config, cfilename).AbsoluteFile), ".nim.c").string
|
||||
mangleModuleName(g.config, cfilename).AbsoluteFile), icCFileExt(g.config)).string
|
||||
let artifact = cfile & ".nif"
|
||||
if not fileExists(artifact):
|
||||
rawMessage(g.config, errGenerated,
|
||||
@@ -873,7 +873,7 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
|
||||
if not decision.broken:
|
||||
var liveOwners = initHashSet[string]()
|
||||
for cname, owner in decision.owners:
|
||||
if owner.endsWith(".c.nif") and cname in decision.live:
|
||||
if owner.endsWith(icCFileExt(g.config) & ".nif") and cname in decision.live:
|
||||
liveOwners.incl owner
|
||||
for owner in liveOwners:
|
||||
let cbase = owner[0 ..< owner.len - ".nif".len] # "@m….nim.c.nif" -> ".c"
|
||||
|
||||
@@ -120,7 +120,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
|
||||
# so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the
|
||||
# driver runs on the exact same config its children will. See icconfig.nim.
|
||||
when not defined(nimKochBootstrap):
|
||||
if conf.cmd in {cmdIc, cmdTrack}:
|
||||
if conf.cmd in {cmdIc, cmdTrack} or isIcDriver(conf):
|
||||
ensureIcConfig(conf)
|
||||
|
||||
var graph = newModuleGraph(cache, conf)
|
||||
|
||||
@@ -29,7 +29,7 @@ const
|
||||
|
||||
nimEnableCovariance* = defined(nimEnableCovariance)
|
||||
|
||||
icFormatVersion* = "36"
|
||||
icFormatVersion* = "37"
|
||||
## Version of the IC cache format (the sem-NIF module layout written by
|
||||
## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
|
||||
## that layout changes: `commandIc` wipes a nimcache whose `ic.version`
|
||||
@@ -940,6 +940,24 @@ proc getOsCacheDir(): string =
|
||||
else:
|
||||
result = getHomeDir() / genSubDir.string
|
||||
|
||||
proc isIcDriver*(conf: ConfigRef): bool =
|
||||
## True for `nim c --ic:on` / `nim cpp --ic:on`: this process is the `nim ic`
|
||||
## DRIVER (it builds the nifmake graph and spawns the per-module children),
|
||||
## not a compilation. `nim ic` itself keeps its own `cmdIc` branch.
|
||||
conf.ic and conf.cmd in {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC}
|
||||
|
||||
proc icCFileExt*(conf: ConfigRef): string =
|
||||
## The extension the per-module backend gives a module's translation unit.
|
||||
## Mirrors `cgen.getCFile` at BACKEND granularity, which is all the `nim ic`
|
||||
## driver can know: it DECLARES every module's `.c`/`.cpp` output to nifmake
|
||||
## without loading a single module, so a per-module `{.compile: cpp.}`
|
||||
## (`sfCompileToCpp`) is out of reach — and `nim cpp` selects the backend for
|
||||
## the whole program anyway.
|
||||
case conf.backend
|
||||
of backendCpp: ".nim.cpp"
|
||||
of backendObjc: ".nim.m"
|
||||
else: ".nim.c"
|
||||
|
||||
proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir =
|
||||
proc nimcacheSuffix(conf: ConfigRef): string =
|
||||
if conf.ideActive: "_nimsuggest" # dedicated cache, never shared with `nim c`
|
||||
|
||||
@@ -2410,6 +2410,7 @@ proc semCppMember(c: PContext; s: PSym; n: PNode) =
|
||||
localError(c.config, n.info, pragmaName & " must be either ptr to object or object type.")
|
||||
if sameOwners(typ.owner, s.owner) and sameOwners(c.module, s.owner):
|
||||
c.graph.memberProcsPerType.mgetOrPut(typ.bindingId, @[]).add s
|
||||
logCppMember(c.graph, s)
|
||||
else:
|
||||
localError(c.config, n.info,
|
||||
pragmaName & " procs must be defined in the same scope as the type they are virtual for and it must be a top level scope")
|
||||
@@ -2432,6 +2433,7 @@ proc semCppMember(c: PContext; s: PSym; n: PNode) =
|
||||
inc j
|
||||
if isInitializer:
|
||||
c.graph.initializersPerType[typ.bindingId] = initializerCall
|
||||
logCppMember(c.graph, s)
|
||||
|
||||
proc semMethodPrototype(c: PContext; s: PSym; n: PNode) =
|
||||
if s.isGenericRoutine:
|
||||
|
||||
75
doc/ic.md
75
doc/ic.md
@@ -2,12 +2,23 @@
|
||||
Incremental Compilation (IC)
|
||||
======================================
|
||||
|
||||
The ``nim ic`` command provides incremental compilation for Nim projects. It
|
||||
decomposes compilation into per-module steps whose results are cached as NIF
|
||||
files, and uses the external ``nifmake`` build tool to re-run only the steps
|
||||
whose inputs changed.
|
||||
``--ic:on`` turns an ordinary compile into an incremental one. It decomposes
|
||||
compilation into per-module steps whose results are cached as NIF files, and
|
||||
uses the external ``nifmake`` build tool to re-run only the steps whose inputs
|
||||
changed.
|
||||
|
||||
This document describes **how `nim ic` works today**, including the edge cases
|
||||
.. code-block:: cmd
|
||||
|
||||
nim c --ic:on myproject.nim
|
||||
nim cpp --ic:on myproject.nim
|
||||
|
||||
It is a switch on the normal compile commands, not a command of its own, so
|
||||
everything else keeps working unchanged: ``cpp`` and ``objc`` backends, ``-r``,
|
||||
``-d:release``, ``--exceptions:``, and a project-wide opt-in from ``nim.cfg`` /
|
||||
``config.nims``. The older spelling ``nim ic`` still works and drives the same
|
||||
code, but it is the C backend only and cannot run the binary it built.
|
||||
|
||||
This document describes **how IC works today**, including the edge cases
|
||||
that shaped the current design. The per-module backend rewrite that earlier
|
||||
editions of this document listed as a *Plan* has **landed**: the whole-program,
|
||||
reuse/redirect/def-retention backend is gone and codegen is now a set of
|
||||
@@ -16,7 +27,7 @@ reuse/redirect/def-retention backend is gone and codegen is now a set of
|
||||
Overview
|
||||
========
|
||||
|
||||
The pipeline has two halves driven by one process (`nim ic`, `commandIc` in
|
||||
The pipeline has two halves driven by one process (the *driver*, `commandIc` in
|
||||
``compiler/deps.nim``) that constructs a dependency graph, writes a build file,
|
||||
and hands it to ``nifmake``:
|
||||
|
||||
@@ -220,7 +231,7 @@ Edge cases (and why the machinery exists)
|
||||
- **Config cost.** Each child re-parsing `nim.cfg` + re-running `config.nims` in
|
||||
the VM was ~80 ms; replaced by a precompiled `ic_config.cfg.nif` replayed in
|
||||
`loadConfigs` (`compiler/icconfig.nim`).
|
||||
- **`koch bootic`** bootstraps the compiler through `nim ic` (a 3-iteration
|
||||
- **`koch bootic`** bootstraps the compiler through `--ic:on` (a 3-iteration
|
||||
fixed-point check). It writes its binary to ``bin/nim_ic`` and never clobbers
|
||||
``bin/nim``.
|
||||
|
||||
@@ -247,7 +258,7 @@ Known residual hack
|
||||
Status and performance
|
||||
======================
|
||||
|
||||
`nim ic` self-builds the compiler (`koch bootic`'s byte-identical fixed-point
|
||||
IC self-builds the compiler (`koch bootic`'s byte-identical fixed-point
|
||||
check) under both `orc` and `--mm:refc`, and passes the external-package CI set.
|
||||
|
||||
Cold full bootstrap on a 32-core box (`-d:release`, **no edits** — IC's worst
|
||||
@@ -256,7 +267,7 @@ case, since incremental reuse is not exercised):
|
||||
| | wall | notes |
|
||||
| - | ---- | ----- |
|
||||
| `koch boot` (classic) | ~1m00s | reference |
|
||||
| `koch bootic` (`nim ic`) | ~1m39s | **~1.66×** |
|
||||
| `koch bootic` (`--ic:on`) | ~1m39s | **~1.66×** |
|
||||
|
||||
This is down from ~7.5× in the whole-program-backend era. IC does modestly more
|
||||
aggregate work (more processes, NIF re-parsing of imports per process), but on a
|
||||
@@ -412,7 +423,7 @@ Testing IC
|
||||
Two mechanisms, at very different scales.
|
||||
|
||||
**`tests/ic` — metamorphic tests.** A `t*.nim` whose body contains `#? metamorphic`
|
||||
drives a sequence of cross-module edits through `nim ic` in one fixed build
|
||||
drives a sequence of cross-module edits through the IC driver in one fixed build
|
||||
directory (see `testament/categories.nim`, `runMetamorphicIcTest`). Directives:
|
||||
|
||||
| directive | effect |
|
||||
@@ -436,11 +447,13 @@ had nowhere to live on a serialized sym node, so every first assignment to a
|
||||
destructor-bearing local became `=sink` over zeroed memory). `koch bootic` has the
|
||||
same blind spot — it proves the compiler reproduces *itself*.
|
||||
|
||||
**`testament --ic` — the whole corpus.** Compiles every C-target test with
|
||||
`nim ic` instead of `nim c`, so IC inherits the existing ~10k programs and their
|
||||
expected output instead of the handful written for it by hand. Tests that
|
||||
override the command (`cmd: "nim c --gc:arc $file"`) are rewritten too, and get a
|
||||
private nimcache; without one they would share a cache and thrash it.
|
||||
**`testament --ic` — the whole corpus.** Appends `--ic:on` to every C and C++
|
||||
test compile, so IC inherits the existing ~10k programs and their expected
|
||||
output instead of the handful written for it by hand. Because it is a switch and
|
||||
not a command, a test that overrides the command wholesale (`cmd: "nim cpp -r
|
||||
$file"`) simply gains the switch — no verb rewriting, and the C++ corpus comes
|
||||
along for free. Each also gets a private nimcache; without one they would share
|
||||
a cache and thrash it.
|
||||
|
||||
To keep that affordable, testament borrows nimony's hastur model
|
||||
(`warmupSharedCache` + `prefillFromWarmup`): a generated warmup program pulling in
|
||||
@@ -458,7 +471,7 @@ Measured on `tests/destructor` (97 test runs, 32-core box):
|
||||
| | cold | warm |
|
||||
| - | ---- | ---- |
|
||||
| `nim c` | 35s | 32s |
|
||||
| `nim ic` | ~3m30 | **9.8s** |
|
||||
| `--ic:on` | ~3m30 | **9.8s** |
|
||||
|
||||
The warm number is the developer loop and it is 3.2x faster than the classic
|
||||
backend; the cold number is paid once per configuration and then cached on disk.
|
||||
@@ -471,3 +484,33 @@ hint, a warning — all of it is produced by the process that actually runs, so
|
||||
build that reuses every artifact prints nothing. Tests that check `nimout` (and
|
||||
anything you are debugging by eye) therefore need a cold cache; running the same
|
||||
test twice in a row makes the second run's `nimout` empty.
|
||||
|
||||
The C++ backend
|
||||
===============
|
||||
|
||||
``nim cpp --ic:on`` works, and `tests/cpp` passes under it. Three things had to
|
||||
change for that, and they are worth knowing because they are the shape of every
|
||||
"C++ needs the whole program" problem the per-module backend has:
|
||||
|
||||
* **The driver must name the right file.** ``deps.nim`` DECLARES each module's
|
||||
translation unit to ``nifmake`` without loading a single module, so it cannot
|
||||
ask ``cgen.getCFile``; ``options.icCFileExt`` mirrors that formula at backend
|
||||
granularity (``.nim.cpp`` / ``.nim.m`` / ``.nim.c``).
|
||||
|
||||
* **C++ has no designated initializers**, so the RTTI record is a bare variable
|
||||
that ``DatInit`` fills field by field. That bare ``TNimTypeV2 x;`` is a
|
||||
tentative definition, which C's linker merges and C++'s does not — every TU
|
||||
that demanded the type defined it. It now gets the same extern-declaration +
|
||||
owned-``'d'``-definition split the C flavour has.
|
||||
|
||||
* **A C++ member is declared inside its class.** ``memberProcsPerType`` and
|
||||
``initializersPerType`` live only in the sem process, so the backend emitted
|
||||
the struct WITHOUT its member declarations; they are replayed from a
|
||||
``(repcppmember …)`` log entry now (``modulegraphs.replayCppMember`` re-derives
|
||||
the type from the routine's signature, exactly as ``semCppMember`` does).
|
||||
Two follow-on details: a member's ``loc.snippet`` is a CALL PATTERN
|
||||
(``#->salute(@)``), so it must be computed even in the TU that only *calls* the
|
||||
member (whole-program cgen got that for free by generating the defining module
|
||||
first), and it is not a linker name — every ``salute`` member in every class
|
||||
mints the same one, so definitions are keyed by their NIF name in the merge
|
||||
stage instead.
|
||||
|
||||
6
koch.nim
6
koch.nim
@@ -76,7 +76,7 @@ Options:
|
||||
--skipIntegrityCheck skips integrity check when booting the compiler
|
||||
Possible Commands:
|
||||
boot [options] bootstraps with given command line options
|
||||
bootic [options] bootstraps via the incremental compiler (`nim ic`)
|
||||
bootic [options] bootstraps via the incremental compiler (`--ic:on`)
|
||||
distrohelper [bindir] helper for distro packagers
|
||||
tools builds Nim related tools
|
||||
toolsNoExternal builds Nim related tools (except external tools,
|
||||
@@ -450,7 +450,7 @@ proc bootic(args: string, skipIntegrityCheck: bool) =
|
||||
# everything.
|
||||
if i > 0: removeDir smartNimcache
|
||||
let nimi = if i == 0: nimStart else: i.thVersion
|
||||
exec "$# ic --nimcache:$# $# compiler" / "nim.nim" %
|
||||
exec "$# c --ic:on --nimcache:$# $# compiler" / "nim.nim" %
|
||||
[nimi, smartNimcache, args]
|
||||
if sameFileContent(output, i.thVersion):
|
||||
copyExe(output, finalDest)
|
||||
@@ -615,7 +615,7 @@ proc runIcTestFile(inp: string) =
|
||||
for fragment in content.split("#!EDIT!#"):
|
||||
let file = inp.replace(".nim", "_temp.nim")
|
||||
writeFile(file, fragment)
|
||||
var cmd = nimExe & " ic --hint:Conf:off --warnings:off "
|
||||
var cmd = nimExe & " c --ic:on --hint:Conf:off --warnings:off "
|
||||
cmd.add quoteShell(file)
|
||||
exec(cmd)
|
||||
|
||||
|
||||
@@ -201,32 +201,31 @@ proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string,
|
||||
var options = target.defaultOptions & ' ' & options
|
||||
if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell)
|
||||
options.add ' ' & extraOptions
|
||||
# `--ic` swaps the C target's command; every other target is left alone (the
|
||||
# incremental compiler has a C backend only).
|
||||
let targetCmd =
|
||||
if useIc and target == targetC: "ic" else: targetToCmd[target]
|
||||
# we avoid using `parseCmdLine` which is buggy, refs bug #14343
|
||||
result = cmdTemplate % ["target", targetCmd,
|
||||
result = cmdTemplate % ["target", targetToCmd[target],
|
||||
"options", options, "file", filename.quoteShell,
|
||||
"filedir", filename.getFileDir(), "nim", compilerPrefix]
|
||||
if useIc and target == targetC:
|
||||
# Roughly half the corpus overrides the command wholesale (`cmd: "nim c
|
||||
# --gc:arc $file"`), which neither goes through `$target` nor picks up
|
||||
# `$options` — so those tests would silently keep using the classic backend
|
||||
# and share one nimcache. Rewrite the compile verb and give them a private
|
||||
# cache, which is what makes them incremental at all.
|
||||
let prefix = compilerPrefix & " c "
|
||||
if result.startsWith(prefix):
|
||||
result = compilerPrefix & " ic " & result[prefix.len .. ^1]
|
||||
if useIc and target in {targetC, targetCpp}:
|
||||
# `--ic:on` turns the ordinary compile command into the IC driver, so the
|
||||
# verb is left alone: roughly half the corpus overrides the command wholesale
|
||||
# (`cmd: "nim c --gc:arc $file"`), which neither goes through `$target` nor
|
||||
# picks up `$options`, and such a test now simply gains the switch. Each also
|
||||
# gets a private nimcache, which is what makes it incremental at all.
|
||||
#
|
||||
# Switches must land BEFORE the project file: anything after it is swallowed
|
||||
# into `config.arguments`, and a non-empty `arguments` without `--run` is a
|
||||
# hard error ("arguments can only be given if the '--run' option is
|
||||
# selected").
|
||||
var switches = "--ic:on "
|
||||
if nimcache.len > 0 and "--nimCache:" notin result and "--nimcache:" notin result:
|
||||
# Must land BEFORE the project file: anything after it is swallowed into
|
||||
# `config.arguments`, and a non-empty `arguments` without `--run` is a hard
|
||||
# error ("arguments can only be given if the '--run' option is selected").
|
||||
let fileArg = filename.quoteShell
|
||||
let at = result.find(fileArg)
|
||||
let switch = "--nimCache:" & nimcache.quoteShell & " "
|
||||
if at >= 0: result = result[0 ..< at] & switch & result[at .. ^1]
|
||||
else: result.add " " & switch
|
||||
switches.add "--nimCache:" & nimcache.quoteShell & " "
|
||||
# `rfind`, not `find`: the private nimcache path embeds the test's file name
|
||||
# (`nimcache/tests/destructor/tmove.nim_<hash>`), so the FIRST occurrence is
|
||||
# inside a switch's value. The project file is the last one.
|
||||
let fileArg = filename.quoteShell
|
||||
let at = result.rfind(fileArg)
|
||||
if at >= 0: result = result[0 ..< at] & switches & result[at .. ^1]
|
||||
else: result.add " " & switches
|
||||
|
||||
proc icWarmupCache(cmdTemplate, filename, options: string, target: TTarget,
|
||||
extraOptions: string): string =
|
||||
@@ -280,7 +279,8 @@ proc prefillIcCache(warmup, nimcache: string) =
|
||||
if warmup.len == 0 or not dirExists(warmup): return
|
||||
if dirExists(nimcache): return # the test already has its own cache
|
||||
const wanted = [".p.nif", ".p.deps.nif", ".deps.nif", ".s.bif", ".iface.bif",
|
||||
".impl.bif", ".edges.bif", ".s.deps.bif", ".t.bif", ".c.nif"]
|
||||
".impl.bif", ".edges.bif", ".s.deps.bif", ".t.bif",
|
||||
".c.nif", ".cpp.nif"]
|
||||
try:
|
||||
createDir(nimcache)
|
||||
for path in walkFiles(warmup / "*"):
|
||||
@@ -299,7 +299,7 @@ proc prefillIcCache(warmup, nimcache: string) =
|
||||
|
||||
proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
|
||||
target: TTarget, extraOptions = ""): TSpec =
|
||||
if useIc and target == targetC and nimcache.len > 0 and not buildingIcWarmup:
|
||||
if useIc and target in {targetC, targetCpp} and nimcache.len > 0 and not buildingIcWarmup:
|
||||
prefillIcCache(icWarmupCache(cmdTemplate, filename, options, target, extraOptions),
|
||||
nimcache)
|
||||
result = TSpec(cmd: prepareTestCmd(cmdTemplate, filename, options, nimcache, target,
|
||||
|
||||
Reference in New Issue
Block a user