mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-08-06 15:29:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

IC: progress (#25879)

Browse Source
This commit is contained in:
Andreas Rumpf
2026-06-14 22:35:06 +02:00
committed by GitHub
parent 9d7c0cc683
commit 7171e6f01f
55 changed files with 4843 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
compiler/ast.nim
View File

@@ -36,6 +36,13 @@ proc setupProgram*(config: ConfigRef; cache: IdentCache) =
when not defined(nimKochBootstrap):
program = createDecodeContext(config, cache)
proc setIcMainModule*(fileIdx: FileIndex) =
## Tells the IC loader which module is being compiled fresh, so that
## re-exports of that module's symbols by dependencies are not loaded as
## duplicate stubs.
when not defined(nimKochBootstrap):
ast2nif.setMainModule(program, fileIdx)
template loadSym(s: PSym) =
## Loads a symbol from NIF file if it's in Partial state.
when not defined(nimKochBootstrap):
@@ -70,6 +77,16 @@ proc backendEnsureMutable*(t: PType) {.inline.} =
# ^ IC review this later
if t.state == Partial: loadType(t)
proc unsealForTransform*(t: PType) {.inline.} =
## The transformer/lambda lifting also run inside `nim m` when the VM
## compiles a LOADED routine (macro evaluation, `getImpl`). Their mutations
## are process-local — transformed bodies are never written back to a NIF —
## so downgrade the loaded type to mutable, mirroring the `cmdNifC` loader
## which loads everything `Complete` for exactly this reason (see
## `ast2nif.loadedState`).
if t.state == Partial: loadType(t)
if t.state == Sealed: t.state = Complete
proc owner*(s: PSym): PSym {.inline.} =
if s.state == Partial: loadSym(s)
result = s.ownerFieldImpl
@@ -221,7 +238,10 @@ proc position*(s: PSym): int {.inline.} =
result = s.positionImpl
proc `position=`*(s: PSym, val: int) {.inline.} =
assert s.state != Sealed
# No `Sealed` guard: the VM reuses `position` as a register slot while compiling
# a macro for execution (see `vmgen.genGenericParams`), which under IC may be a
# macro loaded from a NIF file. The macro is run, not code-generated, so this
# scratch mutation is harmless.
if s.state == Partial: loadSym(s)
s.positionImpl = val
@@ -445,9 +465,13 @@ var gconfig {.threadvar.}: Gconfig
proc setUseIc*(useIc: bool) = gconfig.useIc = useIc
proc comment*(n: PNode): string =
if nfHasComment in n.flags and not gconfig.useIc:
# IC doesn't track comments, see `packed_ast`, so this could fail
result = gconfig.comments[n.nodeId]
if nfHasComment in n.flags:
# NIF-based IC doesn't serialize comments, but the comment table is keyed by
# the node's address (`nodeId`), which is unique among live nodes; a loaded
# node that carries `nfHasComment` simply has no entry here (its comment was
# set in another process), so `getOrDefault` safely returns "" for it while
# in-process VM macro nodes (e.g. newCommentStmtNode) still round-trip.
result = gconfig.comments.getOrDefault(n.nodeId)
else:
result = ""
@@ -478,13 +502,6 @@ proc getPIdent*(a: PNode): PIdent {.inline.} =
of nkOpenSymChoice, nkClosedSymChoice, nkOpenSym: a.sons[0].sym.name
else: nil
const
moduleShift = when defined(cpu32): 20 else: 24
template toId*(a: ItemId): int =
let x = a
(x.module.int shl moduleShift) + x.item.int
template id*(a: PType | PSym): int = toId(a.itemId)
type
@@ -493,28 +510,44 @@ type
symId*: int32
typeId*: int32
sealed*: bool
backendMinted*: bool
disambTable*: CountTable[PIdent]
const
PackageModuleId* = -3'i32
proc idGeneratorFromModule*(m: PSym): IdGenerator =
assert m.kind == skModule
result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0, disambTable: initCountTable[PIdent]())
result.disambTable.inc m.name
proc idGeneratorForBackend*(m: PSym): IdGenerator =
## Like `idGeneratorFromModule`, but for IC codegen (`nim nifc`): symbols and
## types minted fresh during codegen (transf labels/temps, lifted hooks, type
## copies) must not collide with the itemIds the NIF loader synthesizes for
## lazily-loaded symbols/types of the same module — those come from a
## per-module load-order counter that keeps running while codegen mints its
## own ids. A collision corrupts itemId-keyed tables, e.g. `transf`'s inline
## iterator mapping then substitutes a random loaded sym (a call's callee)
## with a `:tmp` block label. Backend-minted ids carry a marker bit in the
## module half (see `itemids.backendItemId`), so the two id spaces are
## disjoint by construction.
assert m.kind == skModule
result = IdGenerator(module: m.itemId.module, symId: 0, typeId: 0,
backendMinted: true, disambTable: initCountTable[PIdent]())
result.disambTable.inc m.name
proc idGeneratorForPackage*(nextIdWillBe: int32): IdGenerator =
result = IdGenerator(module: PackageModuleId, symId: nextIdWillBe - 1'i32, typeId: 0, disambTable: initCountTable[PIdent]())
proc nextSymId(x: IdGenerator): ItemId {.inline.} =
assert(not x.sealed)
inc x.symId
result = ItemId(module: x.module, item: x.symId)
result = if x.backendMinted: backendItemId(x.module, x.symId)
else: itemId(x.module, x.symId)
proc nextTypeId*(x: IdGenerator): ItemId {.inline.} =
assert(not x.sealed)
inc x.typeId
result = ItemId(module: x.module, item: x.typeId)
result = if x.backendMinted: backendItemId(x.module, x.typeId)
else: itemId(x.module, x.typeId)
when false:
proc nextId*(x: IdGenerator): ItemId {.inline.} =
@@ -1043,6 +1076,11 @@ proc newType*(kind: TTypeKind; idgen: IdGenerator; owner: PSym; son: sink PType
if result.itemId.module == 55 and result.itemId.item == 2:
echo "KNID ", kind
writeStackTrace()
when defined(icDbg):
if kind == tyOpenArray:
echo "NEWTYPE openArray id=", id.module, ".", id.item,
" owner=", (if owner != nil: owner.name.s else: "nil")
echo getStackTrace()
proc setSons*(dest: PType; sons: sink seq[PType]) {.inline.} =
assert dest.kind != tyProc or sons.len <= 1
@@ -1105,10 +1143,19 @@ proc copyType*(t: PType, idgen: IdGenerator, owner: PSym): PType =
assignType(result, t)
result.symImpl = t.sym # backend-info should not be copied
proc exactReplica*(t: PType): PType =
proc exactReplica*(t: PType; idgen: IdGenerator): PType =
## Replica that KEEPS `itemId` — the generic-param binding tables
## (`LayeredIdTable`) key on it, so the copy must keep matching its
## original — but mints a FRESH `uniqueId`: uniqueId is the SERIALIZATION
## identity (NIF type names key on it) and must be unique per instance.
## Replicas sharing the original's uniqueId serialized as duplicate defs
## under one NIF name; the loader collapsed them into a single type,
## losing their flag differences (use-site `tfUnresolved` typedescs) or
## their structure (meta instance bodies shadowing a generic's canonical
## body).
result = PType(kind: t.kind, ownerFieldImpl: t.owner, sizeImpl: defaultSize,
alignImpl: defaultAlignment, itemId: t.itemId,
uniqueId: t.uniqueId)
uniqueId: nextTypeId(idgen))
assignType(result, t)
result.symImpl = t.sym # backend-info should not be copied
@@ -1271,6 +1318,9 @@ proc transitionNoneToSym*(n: PNode) =
transitionNodeKindCommon(nkSym)
template transitionSymKindCommon*(k: TSymKind) =
# Under IC the symbol may still be an unloaded stub (`skStub`); materialise it
# first so its kind-specific fields (read below as `obj.*`) actually exist.
if s.state == Partial: loadSym(s)
let obj {.inject.} = s[]
s[] = TSym(kindImpl: k, itemId: obj.itemId, magicImpl: obj.magicImpl, typImpl: obj.typImpl, name: obj.name,
infoImpl: obj.infoImpl, ownerFieldImpl: obj.ownerFieldImpl, flagsImpl: obj.flagsImpl, astImpl: obj.astImpl,

853
compiler/ast2nif.nim
View File

File diff suppressed because it is too large Load Diff

20
compiler/astdef.nim
View File

@@ -20,6 +20,9 @@ export int128
import nodekinds
export nodekinds
import itemids
export itemids
type
TCallingConvention* = enum
ccNimCall = "nimcall" # nimcall, also the default
@@ -571,23 +574,6 @@ const
generatedMagics* = {mNone, mIsolate, mFinished, mOpenArrayToSeq}
## magics that are generated as normal procs in the backend
type
ItemId* = object
module*: int32
item*: int32
proc `$`*(x: ItemId): string =
"(module: " & $x.module & ", item: " & $x.item & ")"
proc `==`*(a, b: ItemId): bool {.inline.} =
a.item == b.item and a.module == b.module
proc hash*(x: ItemId): Hash =
var h: Hash = hash(x.module)
h = h !& hash(x.item)
result = !$h
type
PNode* = ref TNode
TNodeSeq* = seq[PNode]

12
compiler/ccgcalls.nim
View File

@@ -394,7 +394,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; n
# variable. Thus, we create a temporary pointer variable instead.
let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray
if needsIndirect:
n.typ = n.typ.exactReplica
n.typ = n.typ.exactReplica(p.module.idgen)
n.typ.incl tfVarIsPtr
a = initLocExprSingleUse(p, n)
a = withTmpIfNeeded(p, a, needsTmp)
@@ -909,6 +909,16 @@ proc isInactiveDestructorCall(p: BProc, e: PNode): bool =
proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
if p.withinBlockLeaveActions > 0 and isInactiveDestructorCall(p, ri):
return
when defined(icDbgHash):
if ri[0].typ == nil:
echo "NILCALLEE kind=", ri[0].kind,
" sym=", (if ri[0].kind == nkSym: ri[0].sym.name.s else: "-"),
" symKind=", (if ri[0].kind == nkSym: $ri[0].sym.kind else: "-"),
" flags=", (if ri[0].kind == nkSym: $ri[0].sym.flags else: "-"),
" lazy=", nfLazyType in ri[0].flags,
" inProc=", (if p.prc != nil: p.prc.name.s else: "NIL"),
" module=", p.module.module.name.s
raiseAssert "nil callee type, see NILCALLEE above"
if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
genClosureCall(p, le, ri, d)
elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags:

34
compiler/ccgexprs.nim
View File

@@ -3491,7 +3491,23 @@ proc genConstDefinition(q: BModule; p: BProc; sym: PSym) =
data.addDeclWithVisibility(Private):
data.addVarWithInitializer(Local, actualConstName, typ = td):
genBracedInit(q.initProc, sym.astdef, isConst = true, sym.typ, data)
q.s[cfsData].add(extract(data))
if q.config.cmd == cmdNifC:
# Each `cg` process that demands this const emits its definition
# (emit-everywhere). Always declare it first (the data analogue of a proc
# prototype) so a TU whose copy the merge stage drops still has a valid
# declaration; wrap the definition as a droppable `'d'` unit the merge
# stage assigns to a single owner.
let cname = stripCnifMarks(actualConstName)
var decl = newBuilder("")
decl.addDeclWithVisibility(Extern):
decl.addVar(kind = Local, name = actualConstName, typ = td)
q.s[cfsData].add(extract(decl))
q.s[cfsData].add(cnifDefDirective(cname, "d", icNifName(q, sym)))
q.s[cfsData].add(extract(data))
q.s[cfsData].add(cnifEndDefs())
q.icDataDefs.add (cname, icNifName(q, sym))
else:
q.s[cfsData].add(extract(data))
if q.hcrOn:
# generate the global pointer with the real name
q.s[cfsVars].addVar(kind = Global, name = sym.loc.snippet,
@@ -3555,6 +3571,17 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
of skProc, skConverter, skIterator, skFunc:
#if sym.kind == skIterator:
# echo renderTree(sym.getBody, {renderIds})
if p.config.cmd == cmdNifC and
(isGenericRoutineStrict(sym) or sfCompileTime in sym.flags or
(sym.kind == skIterator and sym.typ.callConv == ccInline)):
# Under IC a module's top-level routine definitions are serialized as bare
# symbol references that reappear in the loaded statement list. Uninstantiated
# generic routines (incl. those with type-class params like `tuple`) and
# `.compileTime` routines have no run-time code, so skip them here.
# Inline iterators likewise have no standalone code — they are always inlined
# at their for-loop call sites by the transformer (only closure iterators get
# a standalone C function), so a bare serialized def reference is a no-op.
return
if sfCompileTime in sym.flags:
localError(p.config, n.info, "request to generate code for .compileTime proc: " &
sym.name.s)
@@ -3629,6 +3656,11 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
# echo renderTree(p.prc.ast, {renderIds})
internalError(p.config, n.info, "expr: param not init " & sym.name.s & "_" & $sym.id)
putLocIntoDest(p, d, sym.loc)
of skTemplate, skMacro:
# Under IC a module's top-level template/macro definitions are serialized as
# bare symbol references (only their interface matters), so they reappear in
# the loaded statement list. They are compile-time only and produce no code.
discard
else: internalError(p.config, n.info, "expr(" & $sym.kind & "); unknown symbol")
of nkNilLit:
if not isEmptyType(n.typ):

7
compiler/ccgstmts.nim
View File

@@ -1986,4 +1986,9 @@ proc genStmts(p: BProc, t: PNode) =
if isPush: pushInfoContext(p.config, t.info)
expr(p, t, a)
if isPush: popInfoContext(p.config)
internalAssert p.config, a.k in {locNone, locTemp, locLocalVar, locExpr}
# A bare `nkSym` statement is how IC serializes a definition that lives inside a
# top-level block (e.g. a nested `proc`/`var`): codegen emits the definition and
# leaves the symbol's own location in `a` (e.g. `locProc`), which is discarded
# here, so the value-sanity check below does not apply to it.
internalAssert p.config, t.kind == nkSym or
a.k in {locNone, locTemp, locLocalVar, locExpr}

127
compiler/ccgtypes.nim
View File

@@ -72,6 +72,37 @@ proc mangleProc(m: BModule; s: PSym; makeUnique: bool): string =
else:
m.g.mangledPrcs.incl(result)
proc sharedInstanceCName(m: BModule; s: PSym): string =
## The module-free canonical C name for a content-keyed generic instance,
## or "" when the symbol must keep its module-suffixed name. With a shared
## name, every TU that instantiated the same generic with the same type
## arguments calls one extern definition (first claimant's TU embeds it,
## see `genProcLvl3`) instead of compiling its own static copy.
##
## The name is program-unique only if the 30-bit content hash does not
## collide for same-named instances of *different* instantiations across
## modules — the per-module probe in `setInstanceDisamb` cannot see that.
## Claimants therefore must present the same signature; on mismatch the
## later one keeps its module-suffixed name (no merge, still correct).
## Residual risk: same name and signature, different generic args, AND a
## 30-bit collision — vanishingly unlikely; a full-typeKey verification
## channel can close it later.
result = ""
if m.config.cmd == cmdNifC and s.kind in routineKinds and
(s.disamb and InstanceDisambBit) != 0'i32 and
s.typ != nil and s.typ.callConv != ccInline and not m.hcrOn and
{sfImportc, sfExportc, sfCodegenDecl} * s.flags == {}:
# The content-derived `disamb` is unique per process (collision-probed in
# `setInstanceDisamb`), so the mint-site-independent `_i<disamb>` name is
# safe to use directly; identical instances across modules collide on it
# exactly and the merge stage keeps one.
result = s.name.s.mangle & "_i" & $s.disamb
proc isSharedInstanceCName(m: BModule; s: PSym): bool =
m.config.cmd == cmdNifC and s.kind in routineKinds and
(s.disamb and InstanceDisambBit) != 0'i32 and
stripCnifMarks(s.loc.snippet) == s.name.s.mangle & "_i" & $s.disamb
proc fillBackendName(m: BModule; s: PSym) =
if s.loc.snippet == "":
var result: Rope
@@ -79,13 +110,22 @@ proc fillBackendName(m: BModule; s: PSym) =
m.g.config.symbolFiles == disabledSf:
result = mangleProc(m, s, false).rope
else:
result = s.name.s.mangle.rope
result.add mangleProcNameExt(m.g.graph, s)
let shared = sharedInstanceCName(m, s)
if shared.len > 0:
result = shared.rope
else:
result = s.name.s.mangle.rope
result.add mangleProcNameExt(m.g.graph, s)
if m.hcrOn:
result.add '_'
result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config))
backendEnsureMutable s
s.locImpl.snippet = result
if m.config.cmd == cmdNifC:
# mark the name so the cnif artifact writer can turn every occurrence
# into a Symbol token; stripped from the actual C output in genModule
s.locImpl.snippet = markCName(result)
else:
s.locImpl.snippet = result
proc fillParamName(m: BModule; s: PSym) =
if s.loc.snippet == "":
@@ -373,6 +413,12 @@ proc getSimpleTypeDesc(m: BModule; typ: PType): Rope =
m.typeCache[sig] = result
proc pushType(m: BModule; typ: PType) =
when defined(icDbgRefc):
if typ.kind == tySequence and
typ.elementType.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyGenericParam:
echo "[icRefc] pushType seq-of-genericparam t=", typeToString(typ),
" itemId=", typ.itemId.module, ".", typ.itemId.item, " mod=", m.module.name.s
echo getStackTrace()
for i in 0..high(m.typeStack):
# pointer equality is good enough here:
if m.typeStack[i] == typ: return
@@ -618,6 +664,18 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
for i in 1..<t.n.len:
if t.n[i].kind != nkSym: internalError(m.config, t.n.info, "genProcParams")
var param = t.n[i].sym
# The hidden closure environment param (`:envP`) is not a real C parameter:
# the environment is passed via the trailing `ClE_0` (added below) and
# `closureSetup` materialises `:envP` as a local cast of it. In a from-source
# build `:envP` only lives in the routine's AST params, never in the proc
# *type's* `n`, so it never reaches here. Under IC `closureParams` re-shares
# the AST param node with `typ.n`, so the lifted `:envP` leaks into `t.n`;
# emitting it would produce a bogus extra parameter that collides with the
# `closureSetup` local (the "redeclared as different kind of symbol" / env
# pointer-type mismatch). We still must fill its name/loc (later passes such
# as `assignParam` and `closureSetup` reference it), but it is omitted from
# the C signature to match the from-source ABI.
let isClosureEnv = t.callConv == ccClosure and param.name.s == ":envP"
var descKind = dkParam
if m.config.backend == backendCpp and optByRef in param.options:
if param.typ.kind == tyGenericInst:
@@ -629,6 +687,7 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
fillParamName(m, param)
fillLoc(param.locImpl, locParam, t.n[i],
param.paramStorageLoc)
if isClosureEnv: continue # name/loc filled, but not part of the C signature
var typ: Rope
if ccgIntroducedPtr(m.config, param, t.returnType) and descKind == dkParam:
typ = ptrType(getTypeDescWeak(m, param.typ, check, descKind))
@@ -1108,6 +1167,11 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes
tyUserTypeClass, tyUserTypeClassInst, tyInferred:
result = getTypeDescAux(m, skipModifier(t), check, kind)
else:
when defined(icDbgRefc):
echo "[icRefc] getTypeDescAux ", t.kind, " t=", typeToString(t),
" origTyp=", typeToString(origTyp), " t.itemId=", t.itemId.module, ".", t.itemId.item,
" sym=", (if t.sym != nil: t.sym.name.s else: "nil"),
" owner=", (if t.owner != nil: t.owner.name.s else: "nil")
internalError(m.config, "getTypeDescAux(" & $t.kind & ')')
result = ""
# fixes bug #145:
@@ -1146,6 +1210,10 @@ proc finishTypeDescriptions(m: BModule) =
var check = initIntSet()
while i < m.typeStack.len:
let t = m.typeStack[i]
when defined(icDbgRefc):
echo "[icRefc] finishTypeDescriptions[", i, "] mod=", m.module.name.s,
" t=", typeToString(t), " kind=", t.kind,
" itemId=", t.itemId.module, ".", t.itemId.item
if optSeqDestructors in m.config.globalOptions and t.skipTypes(abstractInst).kind == tySequence:
seqV2ContentType(m, t, check)
else:
@@ -1260,7 +1328,9 @@ proc genProcHeader(m: BModule; prc: PSym; result: var Builder; visibility: var D
elif prc.typ.callConv == ccInline or isNonReloadable(m, prc):
visibility = StaticProc
elif sfImportc notin prc.flags:
visibility = Private
if not isSharedInstanceCName(m, prc):
visibility = Private
# else: plain extern — the definition is shared across TUs
if asPtr:
result.addProcVar(m, prc, name, params, rettype, isStatic = isStaticVar, ignoreAttributes = true)
else:
@@ -1340,6 +1410,8 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
else:
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
if m.config.cmd == cmdNifC:
m.icDataDefs.add (name, icNifName(m, origType))
proc genTypeInfoAux(m: BModule; typ, origType: PType, name: Rope;
info: TLineInfo) =
@@ -1627,8 +1699,13 @@ proc declareNimType(m: BModule; name: string; str: Rope, module: int) =
m.s[cfsTypeInit1].addArgument(hcrGlobal):
m.s[cfsTypeInit1].add("\"" & str & "\"")
else:
# cnif-mark the name: this extern declaration is the reference the
# def-retention check consults when the defining TU regenerates and
# the typeinfo cannot be re-demanded (type vanished) — the referencing
# TU must lose its reuse then instead of producing a link error
let declName = if m.config.cmd == cmdNifC: markCName(str) else: str
m.s[cfsStrData].addDeclWithVisibility(Extern):
m.s[cfsStrData].addVar(kind = Local, name = str, typ = nr)
m.s[cfsStrData].addVar(kind = Local, name = declName, typ = nr)
proc genTypeInfo2Name(m: BModule; t: PType): Rope =
var it = t
@@ -1767,6 +1844,8 @@ proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLin
cgsym(m, "TNimTypeV2")
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
if m.config.cmd == cmdNifC:
m.icDataDefs.add (name, icNifName(m, origType))
var flags = 0
if not canFormAcycle(m.g.graph, t): flags = flags or 1
@@ -1829,8 +1908,15 @@ proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLin
proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) =
cgsym(m, "TNimTypeV2")
m.s[cfsStrData].addDeclWithVisibility(Private):
# Under `nim nifc` every `cg` process that demands this type's RTTI emits its
# definition (emit-everywhere). The forward declaration must therefore be a
# real `extern` (not a tentative definition) so a TU whose copy the merge
# stage drops still only *declares* it; the definition itself is wrapped as a
# droppable `'d'` unit below and assigned to a single owner.
m.s[cfsStrData].addDeclWithVisibility(if m.config.cmd == cmdNifC: Extern else: Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
if m.config.cmd == cmdNifC:
m.icDataDefs.add (name, icNifName(m, origType))
var flags = 0
if not canFormAcycle(m.g.graph, t): flags = flags or 1
@@ -1891,7 +1977,12 @@ proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineIn
else:
typeEntry.addField(typeInit, name = "flags"):
typeEntry.addIntValue(flags)
m.s[cfsVars].add extract(typeEntry)
if m.config.cmd == cmdNifC:
m.s[cfsVars].add(cnifDefDirective(name, "d", icNifName(m, origType)))
m.s[cfsVars].add extract(typeEntry)
m.s[cfsVars].add(cnifEndDefs())
else:
m.s[cfsVars].add extract(typeEntry)
if t.kind == tyObject and t.baseClass != nil and optEnableDeepCopy in m.config.globalOptions:
discard genTypeInfoV1(m, t, info)
@@ -1930,7 +2021,13 @@ proc genTypeInfoV2(m: BModule; t: PType; info: TLineInfo): Rope =
m.typeInfoMarkerV2[sig] = result
let owner = t.skipTypes(typedescPtrs).itemId.module
if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
# In the per-module backend (`cg`) RTTI is emit-everywhere like procs and
# consts: every demanding module emits the `'d'` definition (deduped to one
# owner by the merge stage). The owner-routing below would instead push the
# definition into the owner module's *unwritten* backend module (discarded in
# this process) and emit only an extern here, leaving the symbol undefined.
let perModuleCg = m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"
if not perModuleCg and owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
# make sure the type info is created in the owner module
discard genTypeInfoV2(m.g.mods[owner], origType, info)
# reference the type info as extern here
@@ -1997,6 +2094,10 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
let marker = m.g.typeInfoMarker.getOrDefault(sig)
if marker.str != "":
when defined(icDbgRefc):
if "catchableerror" in marker.str:
echo "[icNti] ", marker.str, " in mod=", m.module.name.s,
" -> extern:globalMarker owner=", marker.owner
cgsym(m, "TNimType")
cgsym(m, "TNimNode")
declareNimType(m, "TNimType", marker.str, marker.owner)
@@ -2007,8 +2108,16 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
result = "NTI$1$2_" % [rope(typeToC(t)), rope($sig)]
m.typeInfoMarker[sig] = result
when defined(icDbgRefc):
template dbgNti(branch: string) =
if "catchableerror" in result:
echo "[icNti] ", result, " in mod=", m.module.name.s, " -> ", branch
else:
template dbgNti(branch: string) = discard
let old = m.g.graph.emittedTypeInfo.getOrDefault($result)
if old != FileIndex(0):
dbgNti "extern:emittedTypeInfo"
cgsym(m, "TNimType")
cgsym(m, "TNimNode")
declareNimType(m, "TNimType", result, old.int)
@@ -2016,6 +2125,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
var owner = t.skipTypes(typedescPtrs).itemId.module
if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
dbgNti "extern:ownerRouted"
# make sure the type info is created in the owner module
discard genTypeInfoV1(m.g.mods[owner], origType, info)
# reference the type info as extern here
@@ -2026,6 +2136,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
else:
owner = m.module.position.int32
dbgNti "DEFINED-HERE"
m.g.typeInfoMarker[sig] = (str: result, owner: owner)
#rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result)

7
compiler/ccgutils.nim
View File

@@ -112,10 +112,13 @@ proc encodeName*(name: string): string =
proc makeUnique(m: BModule; s: PSym, name: string = ""): string =
result = if name == "": s.name.s else: name
# keep backend-minted ids out of the `_u` namespace; their item counter
# restarts at 0 and would collide with loaded symbols' ids
result.add(if s.itemId.isBackendMinted: "_c" else: "_u")
result.add $s.itemId.item
# module suffix LAST (a strippable trailing token; see `mangleProcNameExt`)
result.add "__"
result.add m.g.graph.ifaces[s.itemId.module].uniqueName
result.add "_u"
result.add $s.itemId.item
proc encodeSym*(m: BModule; s: PSym; makeUnique: bool = false; extra: string = ""): string =
#Module::Type

275
compiler/cgen.nim
View File

@@ -19,6 +19,10 @@ import
mangleutils, cbuilderbase, modulegraphs
from expanddefaults import caseObjDefaultBranch
from ast2nif import globalName, toNifFilename, icNifTypeName
from typekeys import modname
from std/algorithm import sort
import cnif
import pipelineutils
@@ -51,7 +55,7 @@ when not declared(dynlib.libCandidates):
else:
dest.add(s)
when options.hasTinyCBackend:
when defined(tinyc): # == hasTinyCBackend; spelled out for the IC dep scanner
import tccgen
proc hcrOn(m: BModule): bool = m.config.hcrOn
@@ -61,9 +65,18 @@ proc addForwardedProc(m: BModule, prc: PSym) =
m.g.forwardedProcs.add(prc)
proc newModule*(g: BModuleList; module: PSym; conf: ConfigRef; idgen: IdGenerator): BModule
proc getCFile*(m: BModule): AbsoluteFile
proc findPendingModule(m: BModule, s: PSym): BModule =
# TODO fixme
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg":
# Per-module backend codegen: only module M (`m`) is emitted in this
# process, so every demanded definition — whether a normal proc owned by
# another (here unwritten) module or a minted instance/hook — is emitted
# into M's TU. Definitions owned elsewhere are emitted again by their own
# module's cg process; the merge stage keeps one per C name and turns the
# rest into prototypes (which already live in the unmarked protos section).
return m
if m.config.symbolFiles == v2Sf or optCompress in m.config.globalOptions:
let ms = s.itemId.module #getModule(s)
result = m.g.mods[ms]
@@ -71,15 +84,52 @@ proc findPendingModule(m: BModule, s: PSym): BModule =
var ms = getModule(s)
registerModule m.g.graph, ms
if ms.position >= m.g.mods.len:
result = newModule(m.g, ms, m.config, idGeneratorFromModule(ms))
result = newModule(m.g, ms, m.config, idGeneratorForBackend(ms))
else:
result = m.g.mods[ms.position]
if result == nil:
result = newModule(m.g, ms, m.config, idGeneratorFromModule(ms))
result = newModule(m.g, ms, m.config, idGeneratorForBackend(ms))
else:
var ms = getModule(s)
result = m.g.mods[ms.position]
proc icNifName(m: BModule; s: PSym): string =
## The serialized NIF name of `s`, recorded next to its C name in the cnif
## artifact so a later run can re-demand the definition when a reused TU
## still references it (the def-retention check). Backend-minted symbols
## have no NIF name.
if m.config.cmd == cmdNifC and s != nil and not isBackendMinted(s.itemId):
result = globalName(s, m.config)
else:
result = ""
proc icNifName(m: BModule; t: PType): string =
## The type flavor: recorded next to RTTI data definitions so the
## def-retention check can re-demand the typeinfo of a regenerating TU's
## previous artifact (`genTypeInfo` is type-driven, not symbol-driven).
if m.config.cmd == cmdNifC:
result = icNifTypeName(t, m.config)
else:
result = ""
proc emitsBodyInThisModule(m: BModule, prc: PSym): bool =
## Per-module backend codegen is concerned with ONE module: it emits the
## bodies of the routines that module OWNS (its own top-level defs) and only
## *prototypes* a routine owned by another module — that routine's body is
## emitted by its own module's `cg` process, and the merge stage's DCE prunes
## whatever ends up globally dead. The funnel where the main module re-emitted
## its entire transitive closure (≈1.8 GB, a 56 MB `.c.nif`) is exactly this
## rule being absent.
##
## Generic instances and synthesized hooks (`=destroy`, `$`, …) have no single
## owning-module top-level — they are minted on demand — so each demander emits
## them and the merge stage deduplicates by their content-addressed C name.
if not (m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"):
return true
result = prc.itemId.module == m.module.position or
(prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32
proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc =
result = TLoc(k: k, storage: s, lode: lode,
snippet: "", flags: flags)
@@ -124,8 +174,6 @@ proc useHeader(m: BModule, sym: PSym) =
proc cgsym(m: BModule, name: string)
proc cgsymValue(m: BModule, name: string): Rope
proc getCFile(m: BModule): AbsoluteFile
proc getModuleDllPath(m: BModule): Rope =
let (dir, name, ext) = splitFile(getCFile(m))
let filename = strutils.`%`(platform.OS[m.g.config.target.targetOS].dllFrmt, [name & ext])
@@ -756,6 +804,9 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
useHeader(p.module, s)
if lfNoDecl in s.loc.flags: return
if not containsOrIncl(p.module.declaredThings, s.id):
if p.config.cmd == cmdNifC and sfImportc notin s.flags:
p.module.icDataDefs.add (stripCnifMarks(s.loc.snippet),
icNifName(p.module, s))
if sfThread in s.flags:
declareThreadVar(p.module, s, sfImportc in s.flags)
if value != "":
@@ -1316,6 +1367,34 @@ proc genProcBody(p: BProc; procBody: PNode) =
p.blocks[0].sections[cpsInit].addCall(cgsymValue(p.module, "nimErrorFlag"))
proc genProcLvl3*(m: BModule, prc: PSym) =
if m.config.cmd == cmdNifC:
fillBackendName(m, prc)
if (prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32 and
containsOrIncl(m.emittedContentDefs, stripCnifMarks(prc.loc.snippet)):
# A different symbol already emitted a body under this content-addressed
# C name in this TU (same generic instance / hook minted in two source
# modules, both loaded here). Emitting a second body is a C redefinition;
# a prototype was already produced for it, so just stop.
return
if sfDispatcher in prc.flags and sfMainModule notin m.module.flags:
# A method dispatcher enumerates the whole program's method set: its
# body is synthesized by `generateIfMethodDispatchers` only after all
# modules have been generated, and its single definition is emitted
# into the main TU by `finishModule` (main is finished last and never
# reused, so the definition can never go stale inside a cached TU).
# Any demand before that point yields a prototype.
genProcPrototype(m, prc)
return
if prc.itemId.module != m.module.position and
not isBackendMinted(prc.itemId) and
(prc.typ == nil or prc.typ.callConv != ccInline) and
sfDispatcher notin prc.flags:
# this TU embeds a definition whose body lives in another module's
# NIF: record the impl dependency (the artifact's cdeps head) so the
# reuse gate re-checks that module's impl cookie. Inline bodies are
# already part of the iface cookie; dispatcher bodies are synthesized
# from the whole program and live in main, which never reuses.
m.icImplMods.incl prc.itemId.module
var p = newProc(prc, m)
var header = newBuilder("")
let isCppMember = m.config.backend == backendCpp and sfCppMember * prc.flags != {}
@@ -1436,7 +1515,37 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
generatedProc.add(extract(p.s(cpsStmts)))
if optStackTrace in prc.options: generatedProc.add(deinitFrame(p))
generatedProc.add(returnStmt)
m.s[cfsProcs].add(extract(generatedProc))
if m.config.cmd == cmdNifC:
# definition directive for the cnif artifact: groups the proc's text
# under its name and carries the root-relevant flags. The end directive
# right after the text makes the definition self-delimiting, so raw
# cfsProcs emitters (NimMain block, trav markers, ...) never end up
# inside a definition's span.
var defFlags = ""
if sfExportc in prc.flags or sfConstructor in prc.flags: defFlags.add 'x'
if sfCompilerProc in prc.flags: defFlags.add 'c'
if prc.kind == skMethod or sfDispatcher in prc.flags: defFlags.add 'm'
if (prc.typ == nil or prc.typ.callConv != ccInline) and
sfDispatcher notin prc.flags:
# A unique program-wide definition: external linkage, so exactly one
# translation unit may embed its body and everyone else declares it.
# Each module's `cg` process emits the body (emit-everywhere); this flag
# tells the merge stage which definitions to assign a single owner and
# prototype in the rest. The complement — inline procs and method
# dispatchers — is emitted into every using TU (`static`/main-only) and
# must never be deduplicated.
defFlags.add 'u'
if not hasCnifMarks(prc.loc.snippet):
# The C name was not minted through `fillBackendName` (e.g. set by an
# `extern`/`rtl` pragma at sem time), so its uses are invisible to the
# artifact's liveness walk — conservatively keep the definition.
defFlags.add 'x'
m.s[cfsProcs].add(cnifDefDirective(stripCnifMarks(prc.loc.snippet), defFlags,
icNifName(m, prc)))
m.s[cfsProcs].add(extract(generatedProc))
m.s[cfsProcs].add(cnifEndDefs())
else:
m.s[cfsProcs].add(extract(generatedProc))
if isReloadable(m, prc):
m.s[cfsDynLibInit].add('\t')
m.s[cfsDynLibInit].addAssignmentWithValue(prc.loc.snippet):
@@ -1482,10 +1591,15 @@ proc genProcPrototype(m: BModule, sym: PSym) =
var header = newBuilder("")
var visibility: DeclVisibility = None
genProcHeader(m, sym, header, visibility, asPtr = asPtr, addAttributes = true)
# A prototype is not a *use*: strip the cnif name marks so the artifact's
# liveness walk does not see every forward-declared proc as referenced.
var headerText = extract(header)
if m.config.cmd == cmdNifC:
headerText = stripCnifMarks(headerText)
if asPtr:
m.s[cfsProcHeaders].addDeclWithVisibility(visibility):
# genProcHeader would give variable declaration, add it directly
m.s[cfsProcHeaders].add(extract(header))
m.s[cfsProcHeaders].add(headerText)
else:
let extraVis =
if sym.typ.callConv != ccInline and requiresExternC(m, sym):
@@ -1494,7 +1608,7 @@ proc genProcPrototype(m: BModule, sym: PSym) =
None
m.s[cfsProcHeaders].addDeclWithVisibility(extraVis):
m.s[cfsProcHeaders].addDeclWithVisibility(visibility):
m.s[cfsProcHeaders].add(extract(header))
m.s[cfsProcHeaders].add(headerText)
m.s[cfsProcHeaders].finishProcHeaderAsProto()
include inliner
@@ -1572,7 +1686,8 @@ proc genProcLvl2(m: BModule, prc: PSym) =
# which will actually become a function pointer
if isReloadable(m, prc):
genProcPrototype(q, prc)
genProcLvl3(q, prc)
if emitsBodyInThisModule(m, prc):
genProcLvl3(q, prc)
else:
fillProcLoc(m, prc.ast[namePos])
useHeader(m, prc)
@@ -1582,7 +1697,7 @@ proc requestConstImpl(p: BProc, sym: PSym) =
if genConstSetup(p, sym):
let m = p.module
# declare implementation:
var q = findPendingModule(m, sym)
let q = findPendingModule(m, sym)
if q != nil and not containsOrIncl(q.declaredThings, sym.id):
assert q.initProc.module == q
genConstDefinition(q, p, sym)
@@ -1606,6 +1721,12 @@ proc genProc(m: BModule, prc: PSym) =
if not containsOrIncl(m.g.generatedHeader.declaredThings, prc.id):
genProcLvl3(m.g.generatedHeader, prc)
proc requestProcDef*(m: BModule, prc: PSym) =
## Public demand entry: request `prc`'s definition; it is routed to the
## module that owns it and generated once, exactly as if some generated
## code had referenced it.
genProc(m, prc)
proc genVarPrototype(m: BModule, n: PNode) =
#assert(sfGlobal in sym.flags)
let sym = n.sym
@@ -1675,7 +1796,7 @@ proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope =
## Returns a mangled module name.
result = mangleModuleName(conf, filename).mangle
proc getSomeNameForModule(m: BModule): Rope =
proc getSomeNameForModule*(m: BModule): Rope =
## Returns a mangled module name.
assert m.module.kind == skModule
assert m.module.owner.kind == skPackage
@@ -2062,6 +2183,40 @@ proc registerModuleToMain(g: BModuleList; m: BModule) =
else:
g.otherModsInit.addCallStmt(init)
proc registerReusedModuleToMain*(g: BModuleList; m: BModule;
initRequired, datInitRequired: bool) =
## `registerModuleToMain` for a module whose cached translation unit is
## reused: the init/datInit presence comes from the artifact's meta head
## instead of the (never generated) sections. Mirrors the non-hcr path of
## `registerModuleToMain` — reuse is disabled when hcr is on.
let
init = m.getInitName
datInit = m.getDatInitName
if datInitRequired:
g.mainModProcs.addDeclWithVisibility(Private):
g.mainModProcs.addProcHeader(ccNimCall, datInit, CVoid, cProcParams())
g.mainModProcs.finishProcHeaderAsProto()
g.mainDatInit.addCallStmt(datInit)
if sfSystemModule in m.module.flags:
if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
g.mainDatInit.addCallStmt(cgsymValue(m, "initThreadVarsEmulation"))
if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcAtomicArc, gcOrc, gcYrc}:
g.mainDatInit.addCallStmt(cgsymValue(m, "initStackBottomWith"),
cCast(CPointer, cAddr("inner")))
if initRequired:
g.mainModProcs.addDeclWithVisibility(Private):
g.mainModProcs.addProcHeader(ccNimCall, init, CVoid, cProcParams())
g.mainModProcs.finishProcHeaderAsProto()
if sfMainModule in m.module.flags:
g.mainModInit.addCallStmt(init)
elif sfSystemModule in m.module.flags:
g.mainDatInit.addCallStmt(init) # systemInit right after systemDatInit
else:
g.otherModsInit.addCallStmt(init)
proc genDatInitCode(m: BModule) =
## this function is called in cgenWriteModules after all modules are closed,
## it means raising dependency on the symbols is too late as it will not propagate
@@ -2309,6 +2464,16 @@ proc genModule(m: BModule, cfile: Cfile): Rope =
moduleIsEmpty = false
res.add(extract(m.s[i]))
# what `registerModuleToMain` will announce for this module; recorded in
# the artifact's meta head so a later run can reuse the TU
let initRequired = m.s[cfsInitProc].buf.len > 0
let datInitRequired = m.s[cfsDatInitProc].buf.len > 0
if m.config.cmd == cmdNifC:
# close the definitions section: the init procs that follow belong to
# the artifact's top level (always-run code, hence liveness roots)
res.add(cnifEndDefs())
if m.s[cfsInitProc].buf.len > 0:
moduleIsEmpty = false
res.add(extract(m.s[cfsInitProc]))
@@ -2331,6 +2496,22 @@ proc genModule(m: BModule, cfile: Cfile): Rope =
postprocessCode(m.config, result)
if m.config.cmd == cmdNifC and result.len > 0:
let artifact = cfile.cname.string & ".nif"
var implDeps: seq[string] = @[]
for pos in m.icImplMods.items:
if pos != m.module.position:
implDeps.add modname(pos, m.config)
sort implDeps
writeCnifArtifact(result, artifact, initRequired, datInitRequired,
m.icDataDefs,
semmedNif = toNifFilename(m.config, FileIndex m.module.position),
moduleBase = getSomeNameForModule(m),
implDeps = implDeps)
m.g.graph.icCnifFiles.add artifact
# NB: under cmdNifC the returned text still carries the cnif marks; the
# caller renders it (dropping dead definitions) or strips it.
proc initProcOptions(m: BModule): TOptions =
let opts = m.config.options
if sfSystemModule in m.module.flags: opts-{optStackTrace} else: opts
@@ -2342,6 +2523,8 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule
result.headerFiles = @[]
result.declaredThings = initIntSet()
result.declaredProtos = initIntSet()
result.emittedContentDefs = initHashSet[string]()
result.icImplMods = initIntSet()
result.cfilename = filename
result.filename = filename
result.typeCache = initTable[SigHash, Rope]()
@@ -2413,10 +2596,13 @@ proc writeHeader(m: BModule) =
result.finishProcHeaderAsProto()
if m.config.cppCustomNamespace.len > 0: closeNamespaceNim(result)
result.addf("#endif /* $1 */$n", [guard])
if not writeRope(extract(result), m.filename):
var headerText = extract(result)
if m.config.cmd == cmdNifC:
headerText = stripCnifMarks(headerText)
if not writeRope(headerText, m.filename):
rawMessage(m.config, errCannotOpenFile, m.filename.string)
proc getCFile(m: BModule): AbsoluteFile =
proc getCFile*(m: BModule): AbsoluteFile =
let ext =
if m.compileToCpp: ".nim.cpp"
elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m"
@@ -2510,8 +2696,9 @@ proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
rawMessage(m.config, errCannotOpenFile, cfile.cname.string)
result = true
proc writeModule(m: BModule) =
let cfile = getCFile(m)
proc genModuleCode(m: BModule; cf: var Cfile): string =
## First half of `writeModule`: finalizes the module and produces its code
## text. Under cmdNifC the text still carries the cnif marks.
if moduleHasChanged(m.g.graph, m.module):
genInitCode(m)
@@ -2526,9 +2713,11 @@ proc writeModule(m: BModule) =
m.s[cfsProcHeaders].add(extract(m.g.mainModProcs))
generateThreadVarsSize(m)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
var code = genModule(m, cf)
result = genModule(m, cf)
proc registerModuleCode(m: BModule; cf: var Cfile; code: string) =
## Second half of `writeModule`: writes the .c file if it changed and
## registers it for compilation.
if code != "" or m.config.symbolFiles != disabledSf:
when hasTinyCBackend:
if m.config.cmd == cmdTcc:
@@ -2538,6 +2727,15 @@ proc writeModule(m: BModule) =
if not shouldRecompile(m, code, cf): cf.flags = {CfileFlag.Cached}
addFileToCompile(m.config, cf)
proc writeModule(m: BModule) =
let cfile = getCFile(m)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
var code = genModuleCode(m, cf)
if m.config.cmd == cmdNifC:
code = stripCnifMarks(code)
registerModuleCode(m, cf, code)
proc updateCachedModule(m: BModule) =
let cfile = getCFile(m)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
@@ -2623,7 +2821,12 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
if m.g.forwardedProcs.len == 0:
incl m.flags, objHasKidsValid
if optMultiMethods in m.g.config.globalOptions or
if m.config.cmd == cmdNifC:
# nifbackend synthesizes the dispatchers between the module loop
# and the finish loop (emitMethodDispatchers): TUs demand-created
# by the dispatcher bodies must still reach `modulesClosed`
discard
elif optMultiMethods in m.g.config.globalOptions or
m.g.config.selectedGC notin {gcArc, gcOrc, gcAtomicArc, gcYrc} or
vtables notin m.g.config.features:
generateIfMethodDispatchers(graph, m.idgen)
@@ -2637,9 +2840,8 @@ proc genForwardedProcs(g: BModuleList) =
# a second pass here
# Note: ``genProcLvl2`` may add to ``forwardedProcs``
while g.forwardedProcs.len > 0:
let
prc = g.forwardedProcs.pop()
m = g.mods[prc.itemId.module]
let prc = g.forwardedProcs.pop()
let m = g.mods[prc.itemId.module]
if sfForward in prc.flags:
internalError(m.config, prc.info, "still forwarded: " & prc.name.s)
@@ -2654,7 +2856,32 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
# order anyway)
genForwardedProcs(g)
for m in cgenModules(g):
m.writeModule()
if config.cmd == cmdNifC and not isDefined(config, "icNoCDce"):
# Two-phase write: produce every module's marked text and artifact
# first, then compute global liveness over the artifacts and render
# the .c files with dead definitions dropped. Demand-driven codegen
# over-approximates (it cannot retract a definition once some path
# requested it); this is where the surplus is removed.
var mods: seq[BModule] = @[]
var cfs: seq[Cfile] = @[]
var codes: seq[string] = @[]
for m in cgenModules(g):
let cfile = getCFile(m)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
let code = genModuleCode(m, cf)
mods.add m
cfs.add cf
codes.add code
let cl = computeLiveFromCArtifacts(g.graph.icCnifFiles)
var dropped = 0
for i in 0..<mods.len:
let rendered =
if cl.broken: stripCnifMarks(codes[i])
else: renderMarkedC(codes[i], cl.live, dropped)
registerModuleCode(mods[i], cfs[i], rendered)
else:
for m in cgenModules(g):
m.writeModule()
writeMapping(config, g.mapping)
if g.generatedHeader != nil: writeHeader(g.generatedHeader)

17
compiler/cgendata.nim
View File

@@ -158,6 +158,12 @@ type
forwTypeCache*: TypeCache # cache for forward declarations of types
declaredThings*: IntSet # things we have declared in this .c file
declaredProtos*: IntSet # prototypes we have declared in this .c file
emittedContentDefs*: HashSet[string]
# cmdNifC per-module backend: content-addressed C names (generic
# instances and synthesized hooks) whose body this TU already emitted.
# Distinct symbols (minted in different source modules) can share one
# `_i<disamb>` name; `declaredThings` keys on symbol id and lets the
# second one through, so we dedup the body by name here instead.
queue*: seq[PSym] # queue of procs to generate
alive*: IntSet # symbol IDs of alive data as computed by `dce.nim`
headerFiles*: seq[string] # needed headers to include
@@ -176,6 +182,17 @@ type
extensionLoaders*: array['0'..'9', Builder] # special procs for the
# OpenGL wrapper
sigConflicts*: CountTable[SigHash]
icImplMods*: IntSet # module ids whose routine BODIES this TU
# embeds (redirected defs, shared instances,
# hooks); recorded as the artifact's cdeps so
# the reuse gate can check their impl cookies
icDataDefs*: seq[tuple[cname, nifname: string]]
# C names of data definitions (consts, globals,
# RTTI) this TU embeds plus their NIF symbol
# names (empty for RTTI, which has no symbol);
# recorded in the cnif artifact so a later run
# can reuse the TU and re-demand definitions
# that cached TUs still reference
g*: BModuleList
template config*(m: BModule): ConfigRef = m.g.config

2
compiler/cgmeth.nim
View File

@@ -180,6 +180,7 @@ proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
g.methods[i].methods[0] != s:
# already exists due to forwarding definition?
localError(g.config, s.info, "method is not a base")
logMethodDef(g, s)
return
of No: discard
of Invalid:
@@ -191,6 +192,7 @@ proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
else:
g.bucketTable.inc(s.typ.firstParamType.skipTypes(skipPtrs).itemId)
g.methods.add((methods: @[s], dispatcher: createDispatcher(s, g, idgen)))
logMethodDef(g, s)
#echo "adding ", s.info
if witness != nil:
localError(g.config, s.info, "invalid declaration order; cannot attach '" & s.name.s &

726
compiler/cnif.nim Normal file
View File

@@ -0,0 +1,726 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## The "cnif" artifact: the C code generator's output as a NIF file.
##
## This is deliberately *not* NIFC: the C text is kept verbatim (Nim's
## C-level machinery — exception handling in particular — is more refined
## than what NIFC models today; the gap can be closed incrementally later).
## The only structure the artifact adds is the part dead code elimination
## and generic-instance merging need:
##
## - raw C text as string literals
## - every *global* entity's C name as a `Symbol` token
## - every emitted proc definition as a `(cdef SymbolDef flags ...)` group
##
## The C generator marks names with control characters at the single place
## a global's C name is minted (`fillBackendName`) and emits a definition
## directive at the single place finished procs are appended; the marks then
## ride through all of the snippet composition untouched. This module turns
## the final marked module text into the `.c.nif` artifact and strips the
## marks for the actual `.c` output. Rendering C from the artifact is a
## plain token walk: string literals verbatim, symbols by name — which is
## also where a later merge step redirects losing generic instances.
##
## Marker scheme (cannot collide: C string literals escape control chars,
## and `\1`/`\31`/`\23` of cgen's postprocess directives are distinct):
## \2 name \3 a global's C name
## \4 name \31 flags \31 nif \5 start of the definition of `name`;
## `nif` is the defining symbol's NIF name
## (empty for backend-minted symbols) so a
## later run can re-demand the definition
## \4 \5 end of the definitions section
import std / [tables, sets, os, assertions, syncio, algorithm]
import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse]
const
CnifSymStart* = '\2'
CnifSymEnd* = '\3'
CnifDefStart* = '\4'
CnifDefSep* = '\31' # same separator char as cgen's postprocess directives
CnifDefEnd* = '\5'
proc markCName*(name: string): string {.inline.} =
CnifSymStart & name & CnifSymEnd
proc hasCnifMarks*(s: string): bool =
for c in s:
if c in {CnifSymStart, CnifSymEnd, CnifDefStart}: return true
false
proc stripCnifMarks*(s: string): string =
## Removes the symbol marks (keeping the names) and the definition
## directives (entirely) so the result is plain C.
if not hasCnifMarks(s): return s
result = newStringOfCap(s.len)
var i = 0
while i < s.len:
case s[i]
of CnifSymStart, CnifSymEnd:
inc i
of CnifDefStart:
while i < s.len and s[i] != CnifDefEnd: inc i
inc i # skip CnifDefEnd
else:
result.add s[i]
inc i
const
CnifVersion* = "4"
## Artifact format version, stored in the meta head. Artifacts written
## by an older compiler lack the NIF names and the cref group the
## def-retention check needs (v2), the cdeps group the fine-grained
## reuse gate needs (v3), or the type NIF names and cnif-marked extern
## RTTI references the typeinfo flavor of the def-retention check
## needs (v4); `readCnifHeads` reports them as invalid so their TUs
## simply regenerate once.
proc cnifDefDirective*(name, flags, nifName: string): string =
CnifDefStart & name & CnifDefSep & flags & CnifDefSep & nifName & CnifDefEnd
proc cnifEndDefs*(): string =
CnifDefStart & CnifDefEnd
proc writeCnifArtifact*(code: string; outfile: string;
initRequired = false; datInitRequired = false;
dataDefs: openArray[tuple[cname, nifname: string]] = [];
semmedNif = ""; moduleBase = "";
implDeps: openArray[string] = []) =
## Splits the marked module text into the `.c.nif` artifact.
## The artifact starts with a `(meta <flags> "semmedNif" "moduleBase"
## "version")` head — whether the module has an init/datInit proc
## ('i'/'d'), which semmed NIF it was generated from and the module's
## mangled base name (what `registerModuleToMain` and the reuse decision
## need when the TU is reused in a later run, possibly without the module
## ever being loaded again) — a `(cdata (SymbolDef StrLit)*)` group naming
## the data definitions (consts, globals, RTTI) the TU embeds together
## with their NIF names, a `(cref Ident*)` group naming every C name
## the TU references but does not define itself (what the def-retention
## check consults when some *other* TU regenerates), and a
## `(cdeps Ident*)` group naming the modules whose routine *bodies* this
## TU embeds (redirected defs, shared instances, hooks): the fine-grained
## reuse gate checks their `.impl.nif` cookies on top of the direct
## imports' `.iface.nif` cookies.
# pre-pass: every marked name is a use, every definition directive (and
# every data def) is a definition; external references = uses - defs
var uses = initHashSet[string]()
var defs = initHashSet[string]()
block prePass:
var i = 0
while i < code.len:
case code[i]
of CnifSymStart:
inc i
var name = ""
while i < code.len and code[i] != CnifSymEnd:
name.add code[i]
inc i
inc i
uses.incl name
of CnifDefStart:
inc i
var payload = ""
while i < code.len and code[i] != CnifDefEnd:
payload.add code[i]
inc i
inc i
let sep = find(payload, CnifDefSep)
if sep > 0: defs.incl payload[0..<sep]
elif payload.len > 0: defs.incl payload
else:
inc i
for d in dataDefs: defs.incl d.cname
var crefs: seq[string] = @[]
for u in uses:
if u notin defs: crefs.add u
sort crefs
var b = nifbuilder.open(outfile)
b.withTree "stmts":
b.withTree "meta":
var metaFlags = ""
if initRequired: metaFlags.add 'i'
if datInitRequired: metaFlags.add 'd'
if metaFlags.len > 0: b.addIdent metaFlags
else: b.addEmpty
b.addStrLit semmedNif
b.addStrLit moduleBase
b.addStrLit CnifVersion
b.withTree "cdata":
for d in dataDefs:
b.addSymbolDef d.cname
b.addStrLit d.nifname
b.withTree "cref":
for r in crefs:
b.addIdent r
b.withTree "cdeps":
for s in implDeps:
b.addIdent s
var raw = ""
var inDef = false
template flushRaw() =
if raw.len > 0:
b.addStrLit raw
raw.setLen 0
var i = 0
while i < code.len:
case code[i]
of CnifSymStart:
flushRaw()
inc i
var name = ""
while i < code.len and code[i] != CnifSymEnd:
name.add code[i]
inc i
inc i # skip CnifSymEnd
b.addSymbol name, ""
of CnifDefStart:
flushRaw()
inc i
var payload = ""
while i < code.len and code[i] != CnifDefEnd:
payload.add code[i]
inc i
inc i # skip CnifDefEnd
if inDef:
b.endTree()
inDef = false
if payload.len > 0:
let sep = find(payload, CnifDefSep)
let name = if sep >= 0: payload[0..<sep] else: payload
var flags = if sep >= 0: payload[sep+1..^1] else: ""
var nifName = ""
let sep2 = find(flags, CnifDefSep)
if sep2 >= 0:
nifName = flags[sep2+1..^1]
flags = flags[0..<sep2]
b.addTree "cdef"
b.addSymbolDef name
if flags.len > 0: b.addIdent flags
else: b.addEmpty
b.addStrLit nifName
inDef = true
else:
raw.add code[i]
inc i
flushRaw()
if inDef:
b.endTree()
b.close()
proc renderMarkedC*(code: string; live: HashSet[string]; dropped: var int): string =
## Renders the final C text from the marked module text: symbol marks are
## removed (keeping the names — a later merge step substitutes them here),
## and definitions whose name is not in `live` are dropped entirely. Each
## definition is self-delimiting (genProcAux emits an end directive right
## after the proc's text), so text written by other emitters is never part
## of a definition's span and survives unconditionally.
result = newStringOfCap(code.len)
var i = 0
while i < code.len:
case code[i]
of CnifSymStart, CnifSymEnd:
inc i
of CnifDefStart:
var payload = ""
inc i
while i < code.len and code[i] != CnifDefEnd:
payload.add code[i]
inc i
inc i # skip CnifDefEnd
if payload.len > 0:
let sep = find(payload, CnifDefSep)
let name = if sep >= 0: payload[0..<sep] else: payload
if name notin live:
inc dropped
# drop the definition's text: everything up to its end directive
while i < code.len and code[i] != CnifDefStart: inc i
else:
result.add code[i]
inc i
# ---- Liveness over the artifact -------------------------------------------
proc symOrIdentName(c: Cursor): string {.inline.} =
if c.kind == Ident: strVal(c) else: symName(c)
type
CnifHeads* = object
## The cheap-to-parse part of an artifact that a later run needs in
## order to reuse the TU without regenerating it.
valid*: bool ## file parsed, carries the meta head and has
## the current format version
initRequired*: bool
datInitRequired*: bool
semmedNif*: string ## the semmed NIF this TU was generated from
moduleBase*: string ## the module's mangled base name
cdefs*: seq[tuple[cname, nifname: string]] ## the proc definitions
cdata*: seq[tuple[cname, nifname: string]] ## the data definitions
crefs*: seq[string] ## C names referenced but not defined here
cdeps*: seq[string] ## module suffixes whose routine bodies this
## TU embeds (impl-cookie gated on reuse)
proc readCnifHeads*(f: string): CnifHeads =
## Reads `(meta ...)`, `(cdata ...)`, `(cref ...)` and the `(cdef ...)`
## head names from an artifact. Artifacts written by an older compiler
## (no meta head or a different format version) report `valid=false`.
result = CnifHeads()
if not fileExists(f): return
var pool = newPool()
var tags = newTagPool()
let stmtsTag = tags.registerTag("stmts")
let cdefTag = tags.registerTag("cdef")
let cdataTag = tags.registerTag("cdata")
let crefTag = tags.registerTag("cref")
let cdepsTag = tags.registerTag("cdeps")
let metaTag = tags.registerTag("meta")
var buf = parseFromFile(f, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return
var version = ""
var sawMeta = false
c.loopInto:
if c.kind == TagLit:
if c.cursorTagId == metaTag:
sawMeta = true
var strIdx = 0
c.loopInto:
if c.kind == Ident:
for ch in strVal(c):
if ch == 'i': result.initRequired = true
elif ch == 'd': result.datInitRequired = true
inc c
elif c.kind == StrLit:
if strIdx == 0: result.semmedNif = strVal(c)
elif strIdx == 1: result.moduleBase = strVal(c)
elif strIdx == 2: version = strVal(c)
inc strIdx
inc c
else:
skip c
elif c.cursorTagId == cdataTag:
c.loopInto:
if c.kind == SymbolDef:
result.cdata.add (symName(c), "")
inc c
elif c.kind == StrLit:
if result.cdata.len > 0:
result.cdata[^1].nifname = strVal(c)
inc c
else:
skip c
elif c.cursorTagId == crefTag:
c.loopInto:
if c.kind in {Ident, Symbol, SymbolDef}:
result.crefs.add symOrIdentName(c)
inc c
else:
skip c
elif c.cursorTagId == cdepsTag:
c.loopInto:
if c.kind in {Ident, Symbol, SymbolDef}:
result.cdeps.add symOrIdentName(c)
inc c
else:
skip c
elif c.cursorTagId == cdefTag:
# fixed head: SymbolDef, flags (Ident or empty), NIF name StrLit;
# everything after that is the definition's body text
var state = 0
c.loopInto:
if c.kind == SymbolDef:
result.cdefs.add (symName(c), "")
state = 1
inc c
elif state == 1: # the flags field
state = 2
skip c
elif state == 2: # the NIF name
if c.kind == StrLit and result.cdefs.len > 0:
result.cdefs[^1].nifname = strVal(c)
state = 3
skip c
else:
skip c
else:
skip c
else:
skip c
endRead(c)
result.valid = sawMeta and version == CnifVersion
type
CnifLiveness* = object
defs*: int ## proc definitions emitted across all modules
liveDefs*: int ## of those, reachable from the roots
live*: HashSet[string] ## live C names
broken*: bool
proc computeLiveFromCArtifacts*(files: openArray[string]): CnifLiveness =
## dce1-style mark&sweep over the C-shaped artifacts: a `(cdef ...)`
## group is a definition (flags 'x'/'c'/'m' — exportc, compilerproc,
## method/dispatcher — make it a root), names at the top level (data,
## globals, init code) are roots, names inside a group are its uses.
## Because the artifact is *fully lowered* output, no conservative
## modelling is needed: every call the C code contains is a token here.
##
## NB: mangled C names contain no dots, so NIF's text reader classifies
## them as `Ident` rather than `Symbol`; the dialect therefore treats
## Ident tokens as name uses. Inside a `(cdef ...)` the flags ident is
## the one immediately following the SymbolDef; everything after is a use.
result = CnifLiveness(live: initHashSet[string]())
var pool = newPool()
var tags = newTagPool()
let stmtsTag = tags.registerTag("stmts")
let cdefTag = tags.registerTag("cdef")
let cdataTag = tags.registerTag("cdata")
let crefTag = tags.registerTag("cref")
let cdepsTag = tags.registerTag("cdeps")
let metaTag = tags.registerTag("meta")
var uses = initTable[string, HashSet[string]]()
var roots = initHashSet[string]()
var defs = initHashSet[string]()
for f in files:
if not fileExists(f):
result.broken = true
return
var buf = parseFromFile(f, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
result.broken = true
endRead(c)
return
c.loopInto:
case c.kind
of Symbol, Ident:
roots.incl symOrIdentName(c)
inc c
of TagLit:
if c.cursorTagId == metaTag or c.cursorTagId == cdataTag or
c.cursorTagId == crefTag or c.cursorTagId == cdepsTag:
# bookkeeping for TU reuse, irrelevant for liveness
skip c
elif c.cursorTagId == cdefTag:
var owner = ""
var flagsSeen = false
c.loopInto:
case c.kind
of SymbolDef:
owner = symName(c)
defs.incl owner
flagsSeen = false
inc c
of Symbol, Ident:
let name = symOrIdentName(c)
if not flagsSeen:
# the flags field right after the SymbolDef
flagsSeen = true
for ch in name:
# 'd' marks a data definition (const/RTTI): never DCE'd, so it
# is a root whose body keeps its referenced procs live
if ch in {'x', 'c', 'm', 'd'}:
roots.incl owner
break
else:
uses.mgetOrPut(owner, initHashSet[string]()).incl name
inc c
of DotToken:
flagsSeen = true # empty flags field
inc c
else:
skip c
else:
c.loopInto:
if c.kind in {Symbol, Ident}:
roots.incl symOrIdentName(c)
inc c
else:
skip c
else:
skip c
endRead(c)
# mark & sweep
var work = newSeqOfCap[string](roots.len)
for r in roots: work.add r
while work.len > 0:
let s = work.pop()
if not result.live.containsOrIncl(s):
if uses.hasKey(s):
for dep in uses[s]:
if dep notin result.live:
work.add dep
result.defs = defs.len
for d in defs:
if d in result.live: inc result.liveDefs
# ---- The merge stage: liveness + owner assignment -------------------------
type
MergeDecision* = object
## What the per-module backend's `merge` stage computes from every
## module's `.c.nif` and what its `emit` stage consumes to render the
## final `.c` of one module.
live*: HashSet[string] ## globally reachable C names (dead cdefs
## are dropped from every module)
owners*: Table[string, string] ## for each `'u'`-flagged (unique,
## externally-linked) definition, the single
## artifact base name allowed to embed its
## body; every other module prototypes it
broken*: bool ## an artifact was missing or unparsable —
## the caller should fall back / regenerate
defs*, liveDefs*: int
proc computeMergeDecision*(files: openArray[string]): MergeDecision =
## One pass over every `.c.nif`: the same mark&sweep as
## `computeLiveFromCArtifacts` plus, per definition, owner assignment.
##
## Each `cg` process emits the body of every definition it demands
## (emit-everywhere), so the same externally-linked definition appears in
## several artifacts. A `'u'` flag on the `(cdef ...)` marks those that need
## exactly one owner, assigned here across processes: the owner is the
## lexicographically smallest artifact that emits it — a pure function of the
## claimant set, hence stable across rebuilds. Definitions without `'u'`
## (inline procs, dispatchers) are `static`/main-only and emitted into every
## using TU, so they get no owner entry and are never deduplicated.
result = MergeDecision(live: initHashSet[string](),
owners: initTable[string, string]())
var pool = newPool()
var tags = newTagPool()
let stmtsTag = tags.registerTag("stmts")
let cdefTag = tags.registerTag("cdef")
let cdataTag = tags.registerTag("cdata")
let crefTag = tags.registerTag("cref")
let cdepsTag = tags.registerTag("cdeps")
let metaTag = tags.registerTag("meta")
var uses = initTable[string, HashSet[string]]()
var roots = initHashSet[string]()
var defs = initHashSet[string]()
for f in files:
if not fileExists(f):
result.broken = true
return
let owner = extractFilename(f)
var buf = parseFromFile(f, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
result.broken = true
endRead(c)
return
c.loopInto:
case c.kind
of Symbol, Ident:
roots.incl symOrIdentName(c)
inc c
of TagLit:
if c.cursorTagId == metaTag or c.cursorTagId == cdataTag or
c.cursorTagId == crefTag or c.cursorTagId == cdepsTag:
skip c
elif c.cursorTagId == cdefTag:
var ownerName = ""
var flagsSeen = false
var needsOwner = false
c.loopInto:
case c.kind
of SymbolDef:
ownerName = symName(c)
defs.incl ownerName
flagsSeen = false
inc c
of Symbol, Ident:
let name = symOrIdentName(c)
if not flagsSeen:
flagsSeen = true
for ch in name:
if ch in {'x', 'c', 'm'}: roots.incl ownerName
# 'u' = unique proc (DCE'd), 'd' = data (never DCE'd, hence a
# root); both need a single owner across the emit-everywhere
# processes
elif ch == 'u': needsOwner = true
elif ch == 'd':
needsOwner = true
roots.incl ownerName
else:
uses.mgetOrPut(ownerName, initHashSet[string]()).incl name
inc c
of DotToken:
flagsSeen = true # empty flags field
inc c
else:
skip c
if needsOwner and ownerName.len > 0:
# smallest claimant wins; ties impossible (one entry per name)
let prev = result.owners.getOrDefault(ownerName, "")
if prev.len == 0 or owner < prev:
result.owners[ownerName] = owner
else:
c.loopInto:
if c.kind in {Symbol, Ident}:
roots.incl symOrIdentName(c)
inc c
else:
skip c
else:
skip c
endRead(c)
var work = newSeqOfCap[string](roots.len)
for r in roots: work.add r
while work.len > 0:
let s = work.pop()
if not result.live.containsOrIncl(s):
if uses.hasKey(s):
for dep in uses[s]:
if dep notin result.live:
work.add dep
result.defs = defs.len
for d in defs:
if d in result.live: inc result.liveDefs
const MergeDecisionFile* = "ic.backend.merge.nif"
## Fixed name of the merge stage's output in the nimcache, read by `emit`.
proc writeMergeDecision*(outfile: string; d: MergeDecision) =
## Serializes the merge decision: `(merge (live Symbol*) (owners (own
## Symbol StrLit)*))`. C names are mangled (no dots) so they serialize as
## symbols; owner artifact base names go in string literals.
var live: seq[string] = @[]
for n in d.live: live.add n
sort live
var keys: seq[string] = @[]
for k in d.owners.keys: keys.add k
sort keys
var b = nifbuilder.open(outfile)
b.withTree "merge":
b.withTree "live":
for n in live: b.addSymbol n, ""
b.withTree "owners":
for k in keys:
b.withTree "own":
b.addSymbol k, ""
b.addStrLit d.owners[k]
b.close()
proc readMergeDecision*(f: string): MergeDecision =
## Reads back a `writeMergeDecision` file; `broken=true` if absent/unparsable.
result = MergeDecision(live: initHashSet[string](),
owners: initTable[string, string]())
if not fileExists(f):
result.broken = true
return
var pool = newPool()
var tags = newTagPool()
let mergeTag = tags.registerTag("merge")
let liveTag = tags.registerTag("live")
let ownersTag = tags.registerTag("owners")
let ownTag = tags.registerTag("own")
var buf = parseFromFile(f, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != mergeTag:
result.broken = true
endRead(c)
return
c.loopInto:
if c.kind == TagLit and c.cursorTagId == liveTag:
c.loopInto:
if c.kind in {Symbol, Ident}:
result.live.incl symOrIdentName(c)
inc c
else:
skip c
elif c.kind == TagLit and c.cursorTagId == ownersTag:
c.loopInto:
if c.kind == TagLit and c.cursorTagId == ownTag:
var key = ""
c.loopInto:
if c.kind in {Symbol, Ident}:
key = symOrIdentName(c)
inc c
elif c.kind == StrLit:
if key.len > 0: result.owners[key] = strVal(c)
inc c
else:
skip c
else:
skip c
else:
skip c
endRead(c)
proc renderCFromArtifact*(artifact: string; d: MergeDecision; ownerId: string;
dropped: var int): string =
## The per-module backend's `emit` stage: render one module's final `.c` from
## its `.c.nif` and the merge decision. String literals are emitted verbatim,
## symbols by name; a `(cdef ...)` body is dropped when the name is dead, or
## when it is a `'u'` unique definition this module does not own. The body's
## prototype lives in the surrounding raw text (cgen emits a forward
## declaration for every *used* proc, independent of where the body lands), so
## a dropped body still leaves a valid declaration — no synthesis needed. The
## head groups (meta/cdata/cref/cdeps) carry no C text.
result = ""
if not fileExists(artifact): return
var pool = newPool()
var tags = newTagPool()
let stmtsTag = tags.registerTag("stmts")
let cdefTag = tags.registerTag("cdef")
var buf = parseFromFile(artifact, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return
c.loopInto:
case c.kind
of StrLit:
result.add strVal(c)
inc c
of Symbol, Ident:
result.add symOrIdentName(c)
inc c
of TagLit:
if c.cursorTagId == cdefTag:
# fixed head: SymbolDef, flags (Ident or empty), nifname StrLit; the
# rest is the definition's body text. `state` counts past the head.
var name = ""
var isUnique = false
var isData = false
var keep = true
var state = 0
c.loopInto:
if state == 0 and c.kind == SymbolDef:
name = symName(c)
state = 1
inc c
elif state == 1: # the flags field (one token: Ident/Symbol or empty)
if c.kind in {Ident, Symbol}:
for ch in symOrIdentName(c):
if ch == 'u': isUnique = true
elif ch == 'd': isData = true
state = 2
inc c
elif state == 2: # the NIF name (one StrLit) — decide keep here
let owned = d.owners.getOrDefault(name, ownerId) == ownerId
keep =
if isData: owned # data: kept by its owner only
elif isUnique: (name in d.live) and owned
else: name in d.live # inline/dispatcher: per-TU
if not keep: inc dropped
state = 3
inc c
else: # body tokens
if keep:
if c.kind == StrLit: result.add strVal(c)
elif c.kind in {Symbol, Ident}: result.add symOrIdentName(c)
inc c
else:
# head groups (meta/cdata/cref/cdeps) carry no C text
skip c
else:
inc c
endRead(c)

52
compiler/commands.nim
View File

@@ -24,7 +24,7 @@ bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
bootSwitch(usedGoGC, defined(gogc), "--gc:go")
bootSwitch(usedNoGC, defined(nogc), "--gc:none")
import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs, enumutils]
import std/[setutils, sets, os, strutils, parseutils, parseopt, sequtils, strtabs, enumutils]
import
msgs, options, nversion, condsyms, extccomp, platform,
wordrecg, nimblecmd, lineinfos, pathutils
@@ -653,6 +653,18 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
conf: ConfigRef) =
var key = ""
var val = ""
# Record config-file switches so the `nim ic` driver can serialise them into a
# precompiled-config artifact and have its per-module child processes replay
# them instead of re-parsing the `nim.cfg` chain (and re-running `config.nims`
# in the VM) on every invocation. Only `passPP` (config-file) switches are
# captured; command-line switches are forwarded by the build graph as usual.
# Path-search switches are skipped: their net effect already lives in the
# resolved `searchPaths` the driver forwards as `--path`, and replaying their
# raw (often relative-to-config-dir) arguments here would misresolve.
if pass == passPP and switch.normalize notin
["path", "p", "nimblepath", "lazypath", "excludepath",
"nonimblepath", "clearnimblepath", "nimcache"]:
conf.icConfigSwitches.add (switch, arg)
case switch.normalize
of "eval":
expectArg(conf, switch, arg, pass, info)
@@ -923,6 +935,44 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
of "noimportdoc":
processOnOffSwitchG(conf, {optNoImportdoc}, arg, pass, info)
of "ismainmodule":
# `nim m` (IC) only: marks the single module being checked as the program's
# real entry point so that `isMainModule` and `when isMainModule:` resolve
# correctly even though every module is compiled with `sfMainModule` set.
conf.isMainModule = switchOn(arg)
of "icgroup":
# `nim m` only: register a module that belongs to the current strongly-
# connected import group, so it is compiled from source (not loaded from a
# precompiled NIF) and gets its own NIF written. `deps.nim` emits one
# `--icGroup:<path>` per member of a dependency cycle. The argument is an
# absolute .nim path produced by the dependency scanner.
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icGroup.incl(canonicalizePath(conf, AbsoluteFile arg).string)
of "icproject":
# `nim m`/`nim nifc` only: the ORIGINAL project file (see options.icProject)
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icProject = canonicalizePath(conf, AbsoluteFile arg).string
of "icpreparsedconfig":
# `nim m`/`nim nifc` only: path of the precompiled-config artifact (see
# options.icPreparsedConfig). Read in `passCmd1`, before `loadConfigs`, so
# config loading can replay it instead of re-parsing the `nim.cfg` chain.
expectArg(conf, switch, arg, pass, info)
conf.icPreparsedConfig = arg
of "icbackendstage":
# `nim nifc` only: per-module backend stage, one of cg|merge|emit (see
# options.icBackendStage). Empty (switch unused) keeps the whole-program
# backend. Emitted by `deps.nim`'s backend build file.
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icBackendStage = arg
of "icbackendmodule":
# `nim nifc` only: the NIF module suffix the cg/emit stage operates on (see
# options.icBackendModule).
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icBackendModule = arg
of "import":
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:

829
compiler/deps.nim
View File

@@ -10,8 +10,9 @@
## Generate a .build.nif file for nifmake from a Nim project.
## This enables incremental and parallel compilation using the `m` switch.
import std / [os, tables, sets, times, osproc]
import options, msgs, lineinfos, pathutils
import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio]
import options, msgs, lineinfos, pathutils, condsyms, icconfig,
modulepaths, extccomp, cnif
import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder]
import "../dist/nimony/src/gear2" / modnames
@@ -33,6 +34,12 @@ type
processedModules: Table[string, int] # modname -> node index
includeStack: seq[string]
systemNodeId: int # ID of the system.nim node
implicitNodeIds: seq[int] # node IDs of `--import`ed modules (conf.implicitImports);
# every ordinary module implicitly imports these, so each
# gets a dependency edge on them, exactly like system.nim
scanningMain: bool # currently scanning the project main module's deps;
# makes `when isMainModule` conditions evaluate true
# only there (every other module is imported)
proc toPair(c: DepContext; f: string): FilePair =
FilePair(nimFile: f, modname: moduleSuffix(f, cast[seq[string]](c.config.searchPaths)))
@@ -46,6 +53,63 @@ proc parsedFile(c: DepContext; f: FilePair): string =
proc semmedFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".nif"
proc ifaceFile(c: DepContext; f: FilePair): string =
## Interface-cookie sidecar written by `nim m` (ast2nif.writeIfaceCookie,
## OnlyIfChanged). Dependents' nim_m rules use it as their input instead of
## the semmed NIF: a body-only change in a dependency then keeps the sidecar
## mtime and nifmake prunes the whole re-sem cascade behind it.
getNimcacheDir(c.config).string / f.modname & ".iface.nif"
proc implFile(c: DepContext; suffix: string): string =
## Implementation-cookie sidecar (ast2nif.writeImplCookie): flips on ANY
## content change of the module (private bodies included; supersedes the
## iface cookie). Used as the edge for dependents that consumed the
## module's bodies at compile time (NeedsImpl edges).
getNimcacheDir(c.config).string / suffix & ".impl.nif"
proc edgesFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".edges.nif"
proc readNeedsImpl(c: DepContext; f: FilePair): seq[string] =
## Reads the module's recorded NeedsImpl edge set (module suffixes whose
## bodies its last sem consumed at compile time). Missing file (never
## compiled yet) -> empty: the rule fires anyway on the first build and the
## recording exists from then on. Recordings are self-correcting with a
## one-run lag: whatever changes a module's consumption set is itself a
## gated input of its rule, so the rule re-fires and re-records.
result = @[]
if fileExists(c.edgesFile(f)):
var s = nifstreams.open(c.edgesFile(f))
try:
discard processDirectives(s.r)
while true:
let t = next(s)
if t.kind == EofToken: break
if t.kind == StringLit:
result.add pool.strings[t.litId]
finally:
close s
proc semDepsFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".s.deps.nif"
proc readSemDeps(c: DepContext; f: FilePair): seq[string] =
## The module's REAL direct imports (full source paths) as sem resolved them,
## including macro-generated imports the static scanner missed
## (ast2nif.writeSemDeps). Missing file (not yet semmed) -> empty.
result = @[]
if fileExists(c.semDepsFile(f)):
var s = nifstreams.open(c.semDepsFile(f))
try:
discard processDirectives(s.r)
while true:
let t = next(s)
if t.kind == EofToken: break
if t.kind == StringLit:
result.add pool.strings[t.litId]
finally:
close s
proc findNifler(): string =
# Look for nifler in common locations
let nimDir = getAppDir()
@@ -63,6 +127,10 @@ proc findNifmake(): string =
proc runNifler(c: DepContext; nimFile: string): bool =
## Run nifler deps on a file if needed. Returns true on success.
## NOTE: the `setLastModificationTime` coordination below is a known hack; its
## clean removal lands with the Phase 2 frontend/backend split, which redefines
## this pre-scan's role. (A naive switch to keying on the parsed file produced
## a stale warm rebuild, so it's left intact until the restructure.)
let pair = c.toPair(nimFile)
let depsPath = c.depsFile(pair)
@@ -78,9 +146,38 @@ proc runNifler(c: DepContext; nimFile: string): bool =
let cmd = quoteShell(c.nifler) & " deps " & quoteShell(nimFile) & " " & quoteShell(depsPath)
let exitCode = execShellCmd(cmd)
result = exitCode == 0
if result:
# The build graph's `nifler parse --deps` rule outputs BOTH the parsed
# file and the deps file. Refreshing the deps file here would MASK that
# rule: nifmake's `needsRebuild` takes the freshest output as proof of
# "ran since the inputs changed", so the rule never re-fires and the
# parsed file goes stale. For an import-cycle group that loses the edit
# entirely — a non-representative member's source is not a direct input
# of the group's `nim_m` rule; its only build-graph connection is the
# (now stale) parsed file. Drop a genuinely stale parsed file so the
# nifler rule re-fires on the missing output.
let parsedPath = c.parsedFile(pair)
if fileExists(parsedPath) and
getLastModificationTime(parsedPath) < getLastModificationTime(nimFile):
removeFile(parsedPath)
# nifler writes OnlyIfChanged: after an edit that leaves the import set
# unchanged the deps file keeps its old mtime and would stay older than
# the source forever, re-running this scan (and re-deleting the parsed
# file) on every warm build. Bump it explicitly: it is the scan's own
# up-to-date marker.
if getLastModificationTime(depsPath) < getLastModificationTime(nimFile):
setLastModificationTime(depsPath, getTime())
proc resolveImport(c: DepContext; origin, toResolve: string): string =
## Resolve an import path using the compiler's normal module lookup rules.
var toResolve = toResolve
if '$' in toResolve:
# string-literal import paths support `$nim`-style substitutions
# (see modulepaths.getModuleName)
try:
toResolve = pathSubs(c.config, toResolve, origin.splitFile().dir)
except ValueError:
discard
result = findModule(c.config, toResolve, origin).string
proc resolveInclude(c: DepContext; origin, toResolve: string): string =
@@ -129,6 +226,12 @@ proc processImport(c: var DepContext; importPath: string; current: Node) =
# Every module depends on system.nim
if c.systemNodeId >= 0:
newNode.deps.add c.systemNodeId
# ... and on every `--import`ed module (conf.implicitImports). A `--import`ed
# module is itself imported by its own closure (which also gets these edges),
# so the cycle folds into one strongly-connected component (see computeSCCs),
# just like system.nim's closure.
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
c.processedModules[pair.modname] = newNode.id
c.nodes.add newNode
traverseDeps(c, pair, newNode)
@@ -148,6 +251,33 @@ proc skipSubtree(s: var Stream; first: PackedToken) =
elif t.kind == ParRi: dec depth
elif t.kind == EofToken: return
proc evalCondIdent(c: DepContext; v: string): bool =
## Truth value of a bare identifier appearing in a `when` condition.
case v
of "false": false
of "hasThreadSupport":
# system.nim's `hasThreadSupport` is `compileOption("threads") and
# not defined(nimscript)`; the conservative `true` would schedule the
# threads-only modules (syslocks, threadtypes, sharedlist, locks)
# whose NIFs a --threads:off compile never produces — nifmake then
# sees missing outputs and re-runs the system rule (and everything
# downstream) on every rerun.
optThreads in c.config.globalOptions
of "usesDestructors":
# system.nim's `usesDestructors = defined(gcDestructors) or
# defined(gcHooks)`; guards mmdisp.nim's `include "system/gc"` whose
# transitive imports (sharedlist, locks) an orc compile never produces.
isDefined(c.config, "gcDestructors") or isDefined(c.config, "gcHooks")
of "isMainModule":
# Only the project main module is compiled with `isMainModule` true; an
# imported module's `when isMainModule` blocks are dead. The conservative
# `true` would schedule main-only imports (e.g. parser.nim's
# `tools/grammar_nanny`, a node that gets a cg rule but is never linked,
# so the merge stage can pick it as a shared def's owner -> undefined
# symbols at link).
c.scanningMain
else: true
proc evalCondExpr(c: DepContext; s: var Stream): bool =
## Read exactly one condition expression from `s` and return its truth
## value. Consumes tokens whether the expression is recognised or not so
@@ -159,10 +289,7 @@ proc evalCondExpr(c: DepContext; s: var Stream): bool =
let t = next(s)
case t.kind
of Ident:
case pool.strings[t.litId]
of "true": result = true
of "false": result = false
else: result = true
result = evalCondIdent(c, pool.strings[t.litId])
of ParLe:
let tag = pool.tags[t.tagId]
case tag
@@ -225,6 +352,19 @@ proc evalCondExpr(c: DepContext; s: var Stream): bool =
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
of "par":
# a parenthesised grouping such as `(defined(a) or defined(b))`: evaluate
# the inner expression. Without this, `par` fell through to the `else`
# branch below and evaluated to `true`, which silently inverted conditions
# like `not (defined(macosx) or defined(bsd))` and dropped real imports
# (e.g. `cpuinfo`'s conditional `import std/posix`).
result = evalCondExpr(c, s)
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
else:
skipSubtree(s, t)
result = true
@@ -298,9 +438,75 @@ proc whenMarkerHolds(c: DepContext; s: var Stream): bool =
# Unknown — treat as true and skip.
skipSubtree(s, t)
elif t.kind == Ident:
let v = pool.strings[t.litId]
if v == "false": result = false
# else (true / unknown ident): keep result
if not evalCondIdent(c, pool.strings[t.litId]): result = false
# a true / unknown ident keeps the current result
proc parseImportPath(s: var Stream; t: var PackedToken): seq[string] =
## Parse an import path expression and return the list of module paths it
## refers to. Handles plain idents (`foo`), string literals, `std/foo`
## infixes (including nested ones like `std/private/since`) and bracketed
## groups like `std/[bitops, fenv]` which expand to several imports.
## On entry `t` is the first token of the expression; on exit `t` is the
## token immediately following the whole expression.
result = @[]
case t.kind
of Ident:
result.add pool.strings[t.litId]
t = next(s)
of StringLit:
result.add pool.strings[t.litId]
t = next(s)
of ParLe:
let tag = pool.tags[t.tagId]
if tag == "infix":
t = next(s) # skip 'infix' tag
var op = ""
if t.kind == Ident:
op = pool.strings[t.litId]
t = next(s)
let left = parseImportPath(s, t)
let right = parseImportPath(s, t)
if op == "as":
# `import ../rlp/results as rlp_results`: the alias is not a path
# component — treating `as` like `/` produced the garbage path
# `../rlp/results/rlp_results`, silently dropping the dependency
result = left
else:
let prefix = if left.len == 1: left[0] else: ""
for r in right:
if prefix.len > 0: result.add prefix & "/" & r
else: result.add r
if t.kind == ParRi: t = next(s) # skip closing ')'
elif tag == "prefix":
# Relative import paths: `import ../dist/checksums/...` parses as
# `(prefix ../ dist)` — a path-prefix operator (`../`, `./`) applied to
# the first path component. Concatenate operator and operand verbatim;
# `findModule` resolves the relative path against the importing module.
t = next(s) # skip 'prefix' tag
var op = ""
if t.kind == Ident:
op = pool.strings[t.litId]
t = next(s)
for r in parseImportPath(s, t):
result.add op & r
if t.kind == ParRi: t = next(s) # skip closing ')'
elif tag == "bracket":
t = next(s) # skip 'bracket' tag
while t.kind != ParRi and t.kind != EofToken:
result.add parseImportPath(s, t)
if t.kind == ParRi: t = next(s) # skip closing ')'
else:
# Unknown subtree: skip it entirely.
var depth = 1
t = next(s)
while depth > 0 and t.kind != EofToken:
if t.kind == ParLe: inc depth
elif t.kind == ParRi: dec depth
if depth == 0: break
t = next(s)
if t.kind == ParRi: t = next(s)
else:
t = next(s)
proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) =
## Read a .deps.nif file and process imports/includes
@@ -308,6 +514,12 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) =
if not fileExists(depsPath):
return
# `current.id == 0` is the project main (rootNode); restored on exit so the
# flag is correct for each parent frame between its child recursions.
let prevScanningMain = c.scanningMain
c.scanningMain = current.id == 0
defer: c.scanningMain = prevScanningMain
var s = nifstreams.open(depsPath)
defer: nifstreams.close(s)
discard processDirectives(s.r)
@@ -323,7 +535,7 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) =
if t.kind == ParLe:
let tag = pool.tags[t.tagId]
case tag
of "import", "fromimport", "include":
of "import", "fromimport", "importexcept", "include":
# Read first child. May be a `(when COND...)` marker — parse and
# evaluate; if the condition is statically false, skip the import
# entirely. Otherwise advance past the marker and parse the path.
@@ -344,33 +556,35 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) =
elif n.kind == EofToken: break
t = next(s)
continue
# Handle path expression (could be ident, string, or infix like std/foo)
var importPath = ""
if t.kind == Ident:
importPath = pool.strings[t.litId]
elif t.kind == StringLit:
importPath = pool.strings[t.litId]
elif t.kind == ParLe and pool.tags[t.tagId] == "infix":
# Handle std / foo style imports
t = next(s) # skip infix tag
if t.kind == Ident: # operator (/)
t = next(s)
if t.kind == Ident: # first part (std)
importPath = pool.strings[t.litId]
t = next(s)
if t.kind == Ident: # second part (foo)
importPath = importPath & "/" & pool.strings[t.litId]
if importPath.len > 0:
if tag == "include":
processInclude(c, importPath, current)
else:
processImport(c, importPath, current)
# Skip to end of node
# Process the path expression(s). Each path supports plain idents,
# string literals, `std/foo` infixes (possibly nested, e.g.
# `std/private/since`) and bracketed groups like `std/[bitops, fenv]`
# that expand to several imports. A plain `import a, b, c` lists several
# modules as siblings; a `fromimport` has a single path followed by the
# imported symbol list, which must not be treated as modules.
if tag == "fromimport" or tag == "importexcept":
# `from m import syms` / `import m except syms`: the first child is the
# module path; the rest is the (in/ex)cluded symbol list, which must not
# be treated as modules. Both still create a real dependency on `m`.
for importPath in parseImportPath(s, t):
if importPath.len > 0:
processImport(c, importPath, current)
else:
while t.kind != ParRi and t.kind != EofToken:
for importPath in parseImportPath(s, t):
if importPath.len > 0:
if tag == "include":
processInclude(c, importPath, current)
else:
processImport(c, importPath, current)
# Drain any remaining tokens of this node (e.g. the symbol list of a
# `fromimport`), up to and including the node's closing ')'.
var depth = 1
while depth > 0:
t = next(s)
while depth > 0 and t.kind != EofToken:
if t.kind == ParLe: inc depth
elif t.kind == ParRi: dec depth
if depth == 0: break
t = next(s)
else:
# Skip unknown node
var depth = 1
@@ -387,11 +601,121 @@ proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) =
return
readDepsFile(c, pair, current)
proc generateBuildFile(c: DepContext): string =
## Generate the .build.nif file for nifmake
proc computeSCCs(c: DepContext): seq[seq[int]] =
## Tarjan's strongly-connected-components over the module dependency graph
## (`node.deps`). Each returned component is a list of node indices; a module
## that is not part of any import cycle yields a singleton component. Tarjan
## emits components in reverse-topological order (a component's external
## dependencies come out before it), which is exactly the order `nifmake`
## needs for the per-group `nim m` build rules.
type Frame = object
v, pi: int
let n = c.nodes.len
var index = newSeq[int](n)
var lowlink = newSeq[int](n)
var onStack = newSeq[bool](n)
var visited = newSeq[bool](n)
var stack: seq[int] = @[]
var counter = 0
result = @[]
# Iterative Tarjan (explicit work stack) so a deep module-dependency chain
# cannot overflow the call stack.
for start in 0..<n:
if visited[start]: continue
var work = @[Frame(v: start, pi: 0)]
while work.len > 0:
let v = work[^1].v
if work[^1].pi == 0:
visited[v] = true
index[v] = counter
lowlink[v] = counter
inc counter
stack.add v
onStack[v] = true
if work[^1].pi < c.nodes[v].deps.len:
let w = c.nodes[v].deps[work[^1].pi]
inc work[^1].pi
if not visited[w]:
work.add Frame(v: w, pi: 0)
elif onStack[w]:
lowlink[v] = min(lowlink[v], index[w])
else:
if lowlink[v] == index[v]:
var comp: seq[int] = @[]
while true:
let w = stack.pop()
onStack[w] = false
comp.add w
if w == v: break
result.add comp
work.setLen work.len - 1
if work.len > 0:
lowlink[work[^1].v] = min(lowlink[work[^1].v], lowlink[v])
proc computeForwardedArgs(c: DepContext): seq[string] =
## Config/define forwarding shared by the frontend (`nim m`) and backend
## (`nim nifc`) child commands. Depends only on the driver's config, not on
## the dependency graph, so it is computed once per `nim ic` run (and also
## writes the precompiled-config artifact the children replay).
##
# Forward the project's configuration to the per-module child processes.
# Non-incremental compilation semchecks every module in one process with one
# define set (the project's config files apply to the stdlib too); the IC
# children compile with the *module* as their project file and would miss
# e.g. compiler/nim.cfg's `define:nimPreviewSlimSystem`, so their `when`
# bodies — and thus their import sets and NIF contents — would silently
# diverge from the dependency graph computed here. Forward every define that
# is not part of the compiler's built-in baseline, plus the threads switch.
let nimcache = getNimcacheDir(c.config).string
result = @[]
let baseline = newStringTable(modeStyleInsensitive)
initDefines(baseline)
for k, v in pairs(c.config.symbols):
if not baseline.hasKey(k) or baseline[k] != v:
result.add "--define:" & k & (if v == "true": "" else: "=" & v)
sort result
result.add "--threads:" & (if optThreads in c.config.globalOptions: "on" else: "off")
# Forward the memory-management mode too: the children would otherwise
# compile with the default GC while the dependency graph here was computed
# with the selected one (e.g. under --mm:refc the scanner keeps
# system/gc's transitive imports but default-orc children never compile
# them — phantom outputs that re-fire the build on every rerun).
if c.config.selectedGC != gcUnselected:
result.add "--mm:" & $c.config.selectedGC
# method dispatch semantics must match across the child processes:
# a child compiled without --multimethods:on builds different dispatch
# buckets (and rejects calls as ambiguous that multi-dispatch accepts)
if optMultiMethods in c.config.globalOptions:
result.add "--multimethods:on"
# the children compile each MODULE as their own project file, which makes
# that module's package the "main package" and unfilters foreign-package
# diagnostics — a vendored package's hintAsError/warningAsError promotions
# then abort builds the whole-program compilation accepts. Forward the
# real project so children filter diagnostics identically.
result.add "--icproject:" & c.config.projectFull.string
# Precompiled config: serialise the driver's config once and have every
# child replay it instead of re-parsing the `nim.cfg` chain and re-running
# `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig`
# restores the old per-child config parsing (for bisecting a suspected
# config-replay divergence without clearing caches).
if not isDefined(c.config, "icNoPreparsedConfig"):
let cfgArtifact = nimcache / "ic_config.cfg.nif"
writeIcConfig(c.config, cfgArtifact)
result.add "--icPreparsedConfig:" & cfgArtifact
proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
## Frontend build file: the nifler (parse) and `nim m` (sem) rules only. The
## driver runs this to a discovery fixpoint; it produces every module's semmed
## NIF plus the cookie/edge sidecars that the backend build file then consumes.
## The backend step lives in its own nifmake run (generateBackendBuildFile) so
## that "which TUs rebuild" stays a pure nifmake mtime decision rather than
## something the driver interleaves with the `.s.deps` discovery loop. This
## split is also the scaffold for the per-module backend: once the backend is
## per-module, its rules slot into the backend file unchanged.
let nimcache = getNimcacheDir(c.config).string
createDir(nimcache)
result = nimcache / c.nodes[0].files[0].modname & ".build.nif"
result = nimcache / c.nodes[0].files[0].modname & ".frontend.build.nif"
var b = nifbuilder.open(result)
defer: b.close()
@@ -420,26 +744,14 @@ proc generateBuildFile(c: DepContext): string =
# Add search paths
for p in c.config.searchPaths:
b.addStrLit "--path:" & p.string
for a in forwardedArgs:
b.addStrLit a
b.addTree "args"
b.endTree()
b.withTree "input":
b.addIntLit 0 # main parsed file
b.endTree()
# Define nim nifc command
b.addTree "cmd"
b.addSymbolDef "nim_nifc"
b.addStrLit getAppFilename()
b.addStrLit "nifc"
b.addStrLit "--nimcache:" & nimcache
# Add search paths
for p in c.config.searchPaths:
b.addStrLit "--path:" & p.string
b.addTree "input"
b.addIntLit 0
b.endTree()
b.endTree()
# Build rules for parsing (nifler)
var seenFiles = initHashSet[string]()
for node in c.nodes:
@@ -459,51 +771,252 @@ proc generateBuildFile(c: DepContext): string =
b.endTree()
b.endTree()
# Build rules for semantic checking (nim m)
for i in countdown(c.nodes.len - 1, 0):
let node = c.nodes[i]
let pair = node.files[0]
# Build rules for semantic checking (nim m).
#
# Modules are grouped into strongly-connected components: a module that is not
# in an import cycle is its own singleton group and compiles in its own
# `nim m <mod>` invocation as before. A cycle (A imports B, B imports A) cannot
# be ordered for separate per-module compilation, so the whole component is
# handed to a single `nim m` invocation: the first member is the project file,
# every member is passed via `--icGroup:<path>` so the compiler compiles them
# all from source in one process (resolving the recursion in-memory) and writes
# a NIF for each. Only dependencies *outside* the component become build-graph
# inputs — intra-component edges are produced by this very rule and listing
# them would reintroduce the cycle nifmake just rejected.
let sccs = computeSCCs(c)
var sccOf = newSeq[int](c.nodes.len)
for sccId, comp in sccs:
for nodeIdx in comp: sccOf[nodeIdx] = sccId
for comp in sccs:
# Representative (project file for this invocation) = smallest node id, so a
# component containing the root (node 0) is driven by the root.
var members = comp
members.sort()
let repPair = c.nodes[members[0]].files[0]
let isGroup = members.len > 1
b.addTree "do"
b.addIdent "nim_m"
# Input: all parsed files for this module
b.withTree "input":
b.addStrLit node.files[0].nimFile
for f in node.files:
b.addTree "input"
b.addStrLit c.parsedFile(f)
b.endTree()
# Also depend on semmed files of dependencies
for depIdx in node.deps:
b.addTree "input"
b.addStrLit c.semmedFile(c.nodes[depIdx].files[0])
b.endTree()
# Output: semmed file
b.addTree "output"
b.addStrLit c.semmedFile(pair)
b.addTree "args"
# The root module (node 0) is the program's real entry point; mark it so
# `isMainModule` resolves to true only for it (every module otherwise gets
# `sfMainModule` for NIF writing under `nim m`).
if members[0] == 0:
b.addStrLit "--isMainModule:on"
# For a real cycle, tell the compiler which modules form the group so it
# compiles them all from source and writes each one's NIF.
if isGroup:
for m in members:
b.addStrLit "--icGroup:" & c.nodes[m].files[0].nimFile
b.endTree()
# Input 0 (the project file passed to `nim m`): the representative's .nim.
b.withTree "input":
b.addStrLit repPair.nimFile
# All parsed files of every member (nifler outputs this group consumes).
for m in members:
for f in c.nodes[m].files:
b.addTree "input"
b.addStrLit c.parsedFile(f)
b.endTree()
# Depend on the dependencies *outside* this component — on their interface
# COOKIE sidecars, not the semmed NIFs themselves: the sidecar's mtime only
# moves when the dep's importer-visible surface (or, via hash chaining, any
# surface in its import closure) changed, so body-only edits stop the
# re-sem cascade right here. Dependencies whose BODIES the last sem of a
# member consumed at compile time (the recorded NeedsImpl edge set) are
# gated on their IMPL cookie instead, which flips on any content change:
# `const x = dep.foo()` then re-sems when foo's body changes.
# `-d:icNoIfaceGate` restores the old full-NIF edges.
let ifaceGate = not isDefined(c.config, "icNoIfaceGate")
var needsImpl = initHashSet[string]()
if ifaceGate:
# union over the members; restricted to the group's transitive dep
# closure: a stale recording naming a module this group no longer
# imports cannot be consumed anymore (and honoring it could even create
# a build-graph cycle after refactorings).
var reachable = initHashSet[string]()
var stack: seq[int] = @[]
for m in members:
for depIdx in c.nodes[m].deps:
if sccOf[depIdx] != sccOf[members[0]]: stack.add depIdx
var visited = initHashSet[int]()
while stack.len > 0:
let n = stack.pop()
if visited.containsOrIncl(n): continue
reachable.incl c.nodes[n].files[0].modname
for depIdx in c.nodes[n].deps: stack.add depIdx
for m in members:
for suffix in readNeedsImpl(c, c.nodes[m].files[0]):
if suffix in reachable: needsImpl.incl suffix
var seenDep = initHashSet[string]()
var directDeps = initHashSet[string]()
for m in members:
for depIdx in c.nodes[m].deps:
if sccOf[depIdx] == sccOf[m]: continue # intra-component edge
let depName = c.nodes[depIdx].files[0].modname
directDeps.incl depName
let depFile =
if not ifaceGate: c.semmedFile(c.nodes[depIdx].files[0])
elif depName in needsImpl: c.implFile(depName)
else: c.ifaceFile(c.nodes[depIdx].files[0])
if not seenDep.containsOrIncl(depFile):
b.addTree "input"
b.addStrLit depFile
b.endTree()
# NeedsImpl on modules that are not direct imports (bodies consumed via
# re-exports or transitively, e.g. a macro's private helper two hops
# away): additional impl-cookie inputs.
if ifaceGate:
var extra: seq[string] = @[]
for suffix in needsImpl:
if suffix notin directDeps: extra.add suffix
sort extra
for suffix in extra:
b.addTree "input"
b.addStrLit c.implFile(suffix)
b.endTree()
# Output: one semmed NIF (plus its cookie/edge sidecars) per member.
for m in members:
b.addTree "output"
b.addStrLit c.semmedFile(c.nodes[m].files[0])
b.endTree()
if ifaceGate:
b.addTree "output"
b.addStrLit c.ifaceFile(c.nodes[m].files[0])
b.endTree()
b.addTree "output"
b.addStrLit c.implFile(c.nodes[m].files[0].modname)
b.endTree()
b.addTree "output"
b.addStrLit c.edgesFile(c.nodes[m].files[0])
b.endTree()
b.endTree()
# Final compilation step: generate executable from main module
b.endTree() # stmts
proc backendCFile(c: DepContext; node: Node): string =
## The `.c` path the backend writes for `node`, computed exactly as
## `cgen.getCFile` does: `mangleModuleName` of the module's cfilename, which
## is the source path for the main module (registered at its source index) and
## the NIF suffix for every dependency (a `fikNifModule` whose `toFullPath` is
## the suffix). Lets nifmake declare a per-module output without loading any
## backend module.
let cfilename =
if node.id == 0: AbsoluteFile node.files[0].nimFile
else: AbsoluteFile node.files[0].modname
result = changeFileExt(completeCfilePath(c.config,
mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string
proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
## Per-module backend build file. One `nim_nifc` command template (the actual
## stage/module switches ride in each rule's `(args …)`), then the stages of
## the per-module backend as separate nifmake rules:
## cg(per module) -> merge -> emit(per module) -> link
## Every module's semmed NIF is a leaf input (produced by the frontend run).
## `cg` emits a module's whole demanded closure into its `.c.nif`
## (emit-everywhere); `merge` picks one owner per duplicated definition across
## all `.c.nif`; `emit` renders each module's `.c` (dropping non-owned/dead
## bodies); `link` compiles and links every `.c` in one `callCCompiler`. The
## main module's `cg` depends on every other `.c.nif` because it reads their
## init/datInit meta heads to wire up NimMain, so it must run last.
let nimcache = getNimcacheDir(c.config).string
createDir(nimcache)
result = nimcache / c.nodes[0].files[0].modname & ".backend.build.nif"
let mainNif = c.nodes[0].files[0].nimFile
let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt)
let mergeFile = nimcache / MergeDecisionFile
# Per-node output paths.
var cnifFiles = newSeq[string](c.nodes.len)
var cFiles = newSeq[string](c.nodes.len)
for i, node in c.nodes:
cFiles[i] = backendCFile(c, node)
cnifFiles[i] = cFiles[i] & ".nif"
var b = nifbuilder.open(result)
defer: b.close()
b.addHeader("nim ic", "nifmake")
b.addTree "stmts"
# Command template: `nifc --nimcache … --path … <forwarded> <per-rule args>
# <project>`. The trailing `(args)` is filled per rule with the stage and
# module switches; `(input 0)` is the project file.
b.addTree "cmd"
b.addSymbolDef "nim_nifc"
b.addStrLit getAppFilename()
b.addStrLit "nifc"
b.addStrLit "--nimcache:" & nimcache
for p in c.config.searchPaths:
b.addStrLit "--path:" & p.string
for a in forwardedArgs:
b.addStrLit a
b.addTree "args"
b.endTree()
b.addTree "input"
b.addIntLit 0
b.endTree()
b.endTree()
template inputStr(s: string) =
b.addTree "input"
b.addStrLit s
b.endTree()
template outputStr(s: string) =
b.addTree "output"
b.addStrLit s
b.endTree()
# cg: one rule per module. Inputs are the project (slot 0) and every semmed
# NIF (so the whole program loads and the rule is ordered after the frontend);
# the main module additionally depends on every other `.c.nif` (init metas).
for i, node in c.nodes:
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:cg"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr mainNif
for n2 in c.nodes:
inputStr c.semmedFile(n2.files[0])
if node.id == 0:
for j in 0 ..< c.nodes.len:
if c.nodes[j].id != 0:
inputStr cnifFiles[j]
outputStr cnifFiles[i]
b.endTree()
# merge: read every `.c.nif`, write the ownership/liveness decision.
b.addTree "do"
b.addIdent "nim_nifc"
# Input: .nim file (expanded as argument)
b.addTree "input"
b.addStrLit mainNif
b.withTree "args":
b.addStrLit "--icBackendStage:merge"
inputStr mainNif
for cn in cnifFiles: inputStr cn
outputStr mergeFile
b.endTree()
# Also depend on the semmed .nif files of the main module and all its
# dependencies. nifmake's topological sort orders nodes by depth; without
# these inputs the nim_nifc node sits at depth 1 (no recognized inputs)
# alongside the nifler nodes and runs *before* the nim_m steps that
# produce the .nif files it needs to read.
for node in c.nodes:
b.addTree "input"
b.addStrLit c.semmedFile(node.files[0])
# emit: render each module's `.c` from its `.c.nif` + the merge decision.
for i, node in c.nodes:
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:emit"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr mainNif
inputStr cnifFiles[i]
inputStr mergeFile
outputStr cFiles[i]
b.endTree()
b.addTree "output"
b.addStrLit exeFile
b.endTree()
# link: compile + link every emitted `.c` in one process.
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:link"
inputStr mainNif
for cf in cFiles: inputStr cf
outputStr exeFile
b.endTree()
b.endTree() # stmts
@@ -521,8 +1034,16 @@ proc commandIc*(conf: ConfigRef) =
rawMessage(conf, errGenerated, "project file not found: " & projectFile)
return
# Create nimcache directory
createDir(getNimcacheDir(conf).string)
# Create nimcache directory; start from a clean one when its format
# stamp is absent or outdated (see `icFormatVersion`)
let cacheDir = getNimcacheDir(conf).string
createDir(cacheDir)
let versionFile = cacheDir & "/ic.version"
let stamp = if fileExists(versionFile): readFile(versionFile) else: ""
if stamp != icFormatVersion:
removeDir(cacheDir)
createDir(cacheDir)
writeFile(versionFile, icFormatVersion)
var c = DepContext(
config: conf,
@@ -540,27 +1061,133 @@ proc commandIc*(conf: ConfigRef) =
c.processedModules[rootPair.modname] = 0
# model the system.nim dependency:
let sysNode = Node(files: @[toPair(c, (conf.libpath / RelativeFile"system.nim").string)], id: 1)
c.nodes.add sysNode
c.systemNodeId = sysNode.id
rootNode.deps.add sysNode.id
let sysPair = toPair(c, (conf.libpath / RelativeFile"system.nim").string)
if sysPair.modname != rootPair.modname:
let sysNode = Node(files: @[sysPair], id: 1)
c.nodes.add sysNode
c.systemNodeId = sysNode.id
rootNode.deps.add sysNode.id
c.processedModules[sysPair.modname] = sysNode.id
# Traverse system.nim's own dependency tree. `nim m system.nim` compiles
# system's entire import closure from source in one process (none of it
# can be precompiled: every module implicitly imports system) and writes
# a NIF for each closure member. Every member also gets the implicit
# dependency edge on system, so Tarjan folds the whole closure into
# system's strongly-connected component and the build file contains a
# single rule producing all of those NIFs. Without this traversal each
# closure member that is also imported by an ordinary module got its own
# `nim m` rule whose output silently OVERWROTE the system-written NIF
# with freshly numbered type ids, leaving dangling type references (the
# ids are baked into sysma2dyk.nif and into every module semchecked
# against the first version) — "symbol has no offset" failures that
# depended on nifmake's scheduling.
traverseDeps(c, sysPair, sysNode)
# Model `--import:X` switches (conf.implicitImports). Every ordinary module
# is compiled with these implicitly imported, so each `nim m` child demands
# the corresponding NIF. They are invisible to the static import scanner
# (they come from config, not from `import` statements) and cannot be
# discovered via `.s.deps` either: every module fails identically at import
# resolution before recording anything, so there is no bootstrap. Seed them
# up front like system.nim — create a node, traverse its closure, and record
# its id so `processImport` adds the edge to every other module. (e.g. Nimbus
# uses `--import:libbacktrace` together with `-d:nimStackTraceOverride`.)
for imp in conf.implicitImports:
let resolved = resolveImport(c, rootPair.nimFile, imp)
if resolved.len == 0 or not fileExists(resolved): continue
let impPair = toPair(c, resolved)
if impPair.modname.len > 0 and impPair.modname notin c.processedModules:
let impNode = Node(files: @[impPair], id: c.nodes.len)
if c.systemNodeId >= 0: impNode.deps.add c.systemNodeId
c.nodes.add impNode
c.processedModules[impPair.modname] = impNode.id
rootNode.deps.add impNode.id
c.implicitNodeIds.add impNode.id
traverseDeps(c, impPair, impNode)
# Process dependencies
traverseDeps(c, rootPair, rootNode)
# Generate build file
let buildFile = generateBuildFile(c)
rawMessage(conf, hintSuccess, "generated: " & buildFile)
# Automatically run nifmake
# Discovery via `.s.deps`: imports GENERATED by macros (chronicles builds
# `import chronicles/textlines` via parseStmt from the chronicles_sinks
# define) are invisible to the static scanner. Each `nim m` records the
# imports it ACTUALLY resolved (static + macro-generated) into a
# `.s.deps.nif` sidecar (ast2nif.writeSemDeps); a child that fails on a
# not-yet-built import flushes it before erroring. We re-derive the graph
# from those sidecars — adding any module the scanner missed, plus the edge
# from its importer — and rerun; nifmake's mtime pruning keeps completed
# work. A round that discovers nothing new but still fails is a real error.
let forwardedArgs = computeForwardedArgs(c)
let nifmake = findNifmake()
if nifmake.len == 0:
rawMessage(conf, hintSuccess, "run: nifmake run " & buildFile)
else:
let cmd = quoteShell(nifmake) & " run " & quoteShell(buildFile)
# Build the per-module rules concurrently: nifmake fans out all commands at
# each DAG depth via execProcesses (defaults to all cores). Cold builds are
# otherwise serial (one child at a time) and leave the machine idle. Opt out
# with `-d:icNoParallel` (e.g. for readable, non-interleaved child output
# when debugging a build).
let parallel = if isDefined(conf, "icNoParallel"): "" else: " --parallel"
# Phase 1 — frontend (nifler + `nim m`), run to a discovery fixpoint.
var rounds = 0
var frontendOk = false
while true:
let buildFile = generateFrontendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & buildFile)
if nifmake.len == 0:
rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & buildFile)
# without nifmake we can only print the manual commands; emit the
# backend's too (best effort — discovery cannot run) and stop.
let backendFile = generateBackendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & backendFile)
rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & backendFile)
return
let cmd = quoteShell(nifmake) & " run" & parallel & " " & quoteShell(buildFile)
rawMessage(conf, hintExecuting, cmd)
let exitCode = execShellCmd(cmd)
if exitCode == 0:
frontendOk = true
break
# Re-derive from the post-sem deps of every node compiled so far. Imports
# the static scanner missed become new nodes; the importer->import edge
# the scanner could not see is added so the discovered module builds
# first. (Static-import edges are already present, so `notin deps` skips
# the redundant ones.)
var discovered = false
inc rounds
if rounds <= 20:
let n0 = c.nodes.len # snapshot: new nodes are traversed as they're added
for ni in 0 ..< n0:
for p in readSemDeps(c, c.nodes[ni].files[0]):
let pair = c.toPair(p)
var idx = c.processedModules.getOrDefault(pair.modname, -1)
if idx == -1:
let newNode = Node(files: @[pair], id: c.nodes.len)
if c.systemNodeId >= 0:
newNode.deps.add c.systemNodeId
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
c.processedModules[pair.modname] = newNode.id
c.nodes.add newNode
idx = newNode.id
traverseDeps(c, pair, newNode)
discovered = true
if idx != ni and idx notin c.nodes[ni].deps:
c.nodes[ni].deps.add idx
discovered = true
if not discovered:
rawMessage(conf, errGenerated, "nifmake failed with exit code: " & $exitCode)
break
# Phase 2 — backend (whole-program `nim nifc`), run once over the now-final
# graph. Kept a separate nifmake run so backend rebuilds are decided purely
# by nifmake's input mtimes, independent of frontend discovery.
if frontendOk:
let backendFile = generateBackendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & backendFile)
let cmd = quoteShell(nifmake) & " run" & parallel & " " & quoteShell(backendFile)
rawMessage(conf, hintExecuting, cmd)
let exitCode = execShellCmd(cmd)
if exitCode != 0:
rawMessage(conf, errGenerated, "nifmake failed with exit code: " & $exitCode)
rawMessage(conf, errGenerated, "nifmake (backend) failed with exit code: " & $exitCode)
else:
rawMessage(conf, errGenerated, "nim ic not available in bootstrap build")

1
compiler/enumtostr.nim
View File

@@ -48,6 +48,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGener
n[resultPos] = newSymNode(res)
result.ast = n
incl result.flagsImpl, {sfFromGeneric, sfNeverRaises}
setHookDisamb(g, result, "$enumtostr", t)
proc searchObjCaseImpl(obj: PNode; field: PSym): PNode =
case obj.kind

34
compiler/ic/replayer.nim
View File

@@ -86,3 +86,37 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) =
g.cacheSeqs[destKey].add val
else:
internalAssert g.config, false
proc replayBackendActions*(g: ModuleGraph; module: PSym; list: PNode) =
## Applies the backend-relevant replay actions (C compile/link directives)
## found in a NIF-loaded module's top-level statement list. The `nifc`
## backend loads modules without going through sem's `replayStateChanges`,
## so e.g. math's `{.passL: "-lm".}` was lost and the final link failed
## with undefined references. VM cache actions are deliberately NOT
## replayed here — codegen does not run macros.
if list == nil: return
for n in list:
if n.kind == nkReplayAction and n.len >= 2 and
n[0].kind == nkStrLit and n[1].kind == nkStrLit:
case n[0].strVal
of "compile":
if n.len == 4 and n[2].kind == nkStrLit:
let cname = AbsoluteFile n[1].strVal
var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
obj: AbsoluteFile n[2].strVal,
flags: {CfileFlag.External},
customArgs: n[3].strVal)
extccomp.addExternalFileToCompile(g.config, cf)
of "link":
extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal)
of "passl":
extccomp.addLinkOption(g.config, n[1].strVal)
of "passc":
extccomp.addCompileOption(g.config, n[1].strVal)
of "localpassc":
extccomp.addLocalCompileOption(g.config, n[1].strVal,
toFullPathConsiderDirty(g.config, module.info.fileIndex))
of "cppdefine":
options.cppDefine(g.config, n[1].strVal)
else:
discard

127
compiler/icconfig.nim Normal file
View File

@@ -0,0 +1,127 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Precompiled config for the incremental compiler (`nim ic`).
##
## `nim ic` builds the program by spawning one `nim m` child per module (or
## strongly-connected import group) plus a final `nim nifc`. Each child is a
## full Nim process, so each would normally re-read the whole `nim.cfg` chain
## *and* re-run `config.nims` through the VM — work that is identical for every
## child and, because of the VM run, far from free. With ~85 modules in the
## compiler itself that config work is paid ~85 times during `koch bootic`.
##
## The fix mirrors Nimony's `.cfg.nif`: the driver parses config once, records
## the net effect, and the children replay it. Every config-file switch funnels
## through `processSwitch(..., passPP, ...)` (`nimconf.parseAssignment` and the
## `switch()` callback in `scriptconfig`), so the recorded sequence of those
## switches, replayed in order, reproduces an identical `ConfigRef` without any
## file read or VM run. The one config side effect that does not go through
## `processSwitch` is `cppDefine` (it mutates `conf.cppDefines` directly), so the
## resolved set is serialised alongside.
##
## Path-search switches are deliberately excluded from the recording (see
## `commands.processSwitch`): their resolved result already lives in
## `conf.searchPaths`, which the driver forwards to every child as absolute
## `--path` arguments; replaying their raw, config-dir-relative arguments here
## would misresolve.
import options, commands, lineinfos
import std/[algorithm, os, sets]
import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse]
const
IcConfigVersion* = "1"
## Artifact format version. Bump on any layout change here so a child built
## by an older compiler rejects a stale artifact and falls back to normal
## config loading instead of replaying a format it cannot parse.
proc writeIcConfig*(conf: ConfigRef; outfile: string) =
## Serialise the config-file switches recorded during `loadConfigs` plus the
## resolved `cppDefines` set into the artifact at `outfile`.
var b = nifbuilder.open(outfile)
b.withTree "stmts":
b.withTree "meta":
b.addStrLit IcConfigVersion
b.withTree "cppdefines":
# HashSet iteration order is unspecified; sort so the artifact is
# byte-stable across runs (nifmake keys rebuilds off content changes).
var defs: seq[string] = @[]
for d in conf.cppDefines: defs.add d
sort defs
for d in defs: b.addStrLit d
b.withTree "switches":
for sw in conf.icConfigSwitches:
b.addTree "sw"
b.addStrLit sw.switch
b.addStrLit sw.arg
b.endTree()
b.close()
proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
## Replay the precompiled config into `conf`. Returns false (and applies
## nothing meaningful) when the artifact is missing or written by a compiler
## with an incompatible format version, so the caller can fall back to reading
## the config files normally.
if not fileExists(infile): return false
var pool = newPool()
var tags = newTagPool()
let
stmtsTag = tags.registerTag("stmts")
metaTag = tags.registerTag("meta")
cppTag = tags.registerTag("cppdefines")
switchesTag = tags.registerTag("switches")
swTag = tags.registerTag("sw")
var buf = parseFromFile(infile, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return false
var version = ""
var sawMeta = false
let info = unknownLineInfo
c.loopInto:
if c.kind == TagLit:
if c.cursorTagId == metaTag:
sawMeta = true
c.loopInto:
if c.kind == StrLit:
version = strVal(c)
inc c
else:
skip c
elif c.cursorTagId == cppTag:
c.loopInto:
if c.kind == StrLit:
cppDefine(conf, strVal(c))
inc c
else:
skip c
elif c.cursorTagId == switchesTag:
c.loopInto:
if c.kind == TagLit and c.cursorTagId == swTag:
var sw = ""
var arg = ""
var idx = 0
c.loopInto:
if c.kind == StrLit:
if idx == 0: sw = strVal(c)
else: arg = strVal(c)
inc idx
inc c
else:
skip c
processSwitch(sw, arg, passPP, info, conf)
else:
skip c
else:
skip c
else:
skip c
endRead(c)
result = sawMeta and version == IcConfigVersion

98
compiler/itemids.nim Normal file
View File

@@ -0,0 +1,98 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## `ItemId` is the identity of a symbol or type: a `(module, item)` pair.
##
## The fields are private on purpose: the module half reserves bit 30 as the
## "backend minted" marker, so all construction and inspection has to go
## through this module's API and the marker bit can never leak into module
## indexing or arithmetic.
##
## Three id spaces coexist per module:
## - Semantic-phase and NIF-loader ids: `itemId(module, item)` with `item > 0`.
## - Backend-minted ids (IC codegen, `nim nifc`: transf labels and temps,
## lifted hooks): `backendItemId` sets `BackendModuleBit`, so these can
## never compare equal to a loader id even though both counters mint the
## same small `item` range in one process. They never cross a process
## boundary and must never be written to a NIF file.
## - Derived env/tuple-field ids (`lowerings.addField`): the source local's
## id with `item` negated. `derivedFieldId` preserves the backend marker,
## keeping the derivation collision-free for both id spaces above.
import std/hashes
when defined(nimPreviewSlimSystem):
import std/assertions
const
BackendModuleBit = 0x4000_0000'i32
# Bit 30 of the module field. Bit 31 stays clear so marked module values
# remain non-negative and cannot be mistaken for the special negative
# module ids like `PackageModuleId`.
PackageModuleId* = -3'i32
type
ItemId* = object
moduleBits: int32
itemBits: int32
proc itemId*(module, item: int32): ItemId {.inline.} =
assert module < 0 or (module and BackendModuleBit) == 0
ItemId(moduleBits: module, itemBits: item)
proc backendItemId*(module, item: int32): ItemId {.inline.} =
## An id minted during IC codegen; distinct from every `itemId` of the
## same module so that the loader's stub counter and the backend's counter
## cannot collide in id-keyed tables.
assert module >= 0 and (module and BackendModuleBit) == 0
ItemId(moduleBits: module or BackendModuleBit, itemBits: item)
proc module*(x: ItemId): int32 {.inline.} =
if x.moduleBits >= 0: x.moduleBits and not BackendModuleBit
else: x.moduleBits
proc item*(x: ItemId): int32 {.inline.} = x.itemBits
proc isBackendMinted*(x: ItemId): bool {.inline.} =
x.moduleBits >= 0 and (x.moduleBits and BackendModuleBit) != 0
proc derivedFieldId*(source: ItemId): ItemId {.inline.} =
## The id of the env/tuple field that `lowerings.addField` derives for a
## captured local: `item` negated, module bits (including the backend
## marker) preserved.
ItemId(moduleBits: source.moduleBits, itemBits: -abs(source.itemBits))
proc matchesDerivedFieldId*(field, source: ItemId): bool {.inline.} =
## Does `field` carry the id `derivedFieldId` would derive for `source`?
## `source` may itself already be the derived field id.
field.moduleBits == source.moduleBits and
field.itemBits == -abs(source.itemBits)
proc `==`*(a, b: ItemId): bool {.inline.} =
# raw bit comparison: a backend-minted id never equals a loader id
a.itemBits == b.itemBits and a.moduleBits == b.moduleBits
proc hash*(x: ItemId): Hash =
var h: Hash = hash(x.moduleBits)
h = h !& hash(x.itemBits)
result = !$h
proc `$`*(x: ItemId): string =
result = "(module: " & $x.module & ", item: " & $x.itemBits
if x.isBackendMinted: result.add ", backend"
result.add ")"
const
moduleShift = when defined(cpu32): 20 else: 24
proc toId*(a: ItemId): int {.inline.} =
## Packs an ItemId into a single int. Uses the raw module bits so the
## backend marker keeps the two id spaces disjoint (bit 30 shifts to
## bit 54; like the module/item split itself this needs a 64-bit int).
(a.moduleBits.int shl moduleShift) + a.itemBits.int

18
compiler/lambdalifting.nim
View File

@@ -164,9 +164,21 @@ proc getClosureIterResult*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym
incl(result.flagsImpl, sfUsed)
iter.ast.add newSymNode(result)
proc closureParams(routine: PSym): PNode =
## The formal parameters node lambda lifting reads and extends. In a
## from-source compilation `routine.ast[paramsPos]` and `routine.typ.n` are the
## very same node (see the `typ.n.len` based position math below). Under IC the
## loaded proc AST omits the parameters (they are kept only in `typ.n`), so
## restore the shared node here.
result = routine.ast[paramsPos]
if (result == nil or result.kind == nkEmpty) and routine.typ != nil and
routine.typ.n != nil and routine.ast.len > paramsPos:
result = routine.typ.n
routine.ast[paramsPos] = result
proc addHiddenParam(routine: PSym, param: PSym) =
assert param.kind == skParam
var params = routine.ast[paramsPos]
var params = closureParams(routine)
# -1 is correct here as param.position is 0 based but we have at position 0
# some nkEffect node:
param.position = routine.typ.n.len-1
@@ -177,7 +189,8 @@ proc addHiddenParam(routine: PSym, param: PSym) =
proc getEnvParam*(routine: PSym): PSym =
if routine.ast.isNil: return nil
let params = routine.ast[paramsPos]
let params = closureParams(routine)
if params == nil or params.len == 0: return nil
let hidden = lastSon(params)
if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName:
result = hidden.sym
@@ -294,6 +307,7 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
elif not (owner.typ.isClosure or owner.isNimcall and not owner.isExplicitCallConv or isEnv):
localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
[s.name.s, owner.name.s, $owner.typ.callConv])
unsealForTransform(owner.typ)
incl(owner.typ, tfCapturesEnv)
if not isEnv:
owner.typ.callConv = ccClosure

22
compiler/liftdestructors.nim
View File

@@ -711,6 +711,11 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
doAssert t.asink != nil
body.add newHookCall(c, t.asink, x, y)
of attachedDestructor:
when defined(icDbg):
if t.destructor == nil:
echo "MISSING destructor: ", typeToString(t), " kind=", t.kind,
" itemId=", t.itemId, " uniqueId=", t.uniqueId, " state=", t.state,
" owner=", (if t.owner != nil: t.owner.name.s else: "nil")
doAssert t.destructor != nil
body.add destructorCall(c, t.destructor, x)
of attachedTrace:
@@ -1080,8 +1085,17 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
case t.kind
of tyNone, tyEmpty, tyVoid: discard
of tyUncheckedArray:
# An UncheckedArray has no known length, so it cannot be copied, moved or
# destroyed as a value: it only ever lives behind a pointer and its bytes
# are managed manually (element ops for seqs/strings go through the
# seq/string hooks, which know the length). Emitting `x = y` for it (as the
# pointer-like group below does) produces an assignment of an unsized array,
# which the C backend cannot lower (genAssignment: tyUncheckedArray). So all
# value hooks for it are no-ops.
discard
of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring,
tyPtr, tyUncheckedArray, tyVar, tyLent:
tyPtr, tyVar, tyLent:
defaultOp(c, t, body, x, y)
of tyRef:
if c.g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc}:
@@ -1221,6 +1235,7 @@ proc symDupPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttache
n[resultPos] = newSymNode(res)
result.ast = n
incl result.flagsImpl, {sfFromGeneric, sfGeneratedOp}
setHookDisamb(g, result, AttachedOpToStr[kind], typ)
proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
info: TLineInfo; idgen: IdGenerator; isDiscriminant = false): PSym =
@@ -1267,6 +1282,10 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
if kind == attachedWasMoved:
incl result.flagsImpl, sfNoSideEffect
incl result.typ, tfNoSideEffect
if not isDiscriminant:
# discriminant destructors derive their body from the enclosing object
# AND the selected field; their key is set at the call site
setHookDisamb(g, result, AttachedOpToStr[kind], typ)
proc genTypeFieldCopy(c: var TLiftCtx; t: PType; body, x, y: PNode) =
let xx = genBuiltin(c, mAccessTypeField, "accessTypeField", x)
@@ -1359,6 +1378,7 @@ proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym,
assert(typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject)
# discrimantor assignments needs pointers to destroy fields; alas, we cannot use non-var destructor here
result = symPrototype(g, field.typ, typ.owner, attachedDestructor, info, idgen, isDiscriminant = true)
setHookDisamb(g, result, "=destroy¦" & field.name.s & "¦" & $field.position, typ)
var a = TLiftCtx(info: info, g: g, kind: attachedDestructor, asgnForType: typ, idgen: idgen,
fn: result)
a.asgnForType = typ

3
compiler/lookups.nim
View File

@@ -378,6 +378,9 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
conflictsWith: TLineInfo, note = errGenerated) =
## Emit a redefinition error if in non-interactive mode
if c.config.cmd != cmdInteractive:
when defined(icDbgRefc):
echo "[icRedef] ", s
echo getStackTrace()
localError(c.config, info, note,
"redefinition of '$1'; previous declaration here: $2" %
[s, c.config $ conflictsWith])

6
compiler/lowerings.nim
View File

@@ -207,7 +207,7 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym =
if result != nil: return
else: discard
of nkSym:
if n.sym.itemId.module == id.module and n.sym.itemId.item == -abs(id.item): result = n.sym
if matchesDerivedFieldId(n.sym.itemId, id): result = n.sym
else: discard
proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym =
@@ -215,7 +215,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym
# This is hacky but the clean solution is much more complex than it looks.
var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
idgen, s.owner, s.info, s.options)
field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
field.itemId = derivedFieldId(s.itemId)
let t = skipIntLit(s.typ, idgen)
field.typ = t
if s.kind in {skLet, skVar, skField, skForVar}:
@@ -235,7 +235,7 @@ proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator)
if result == nil:
var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), idgen,
s.owner, s.info, s.options)
field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
field.itemId = derivedFieldId(s.itemId)
let t = skipIntLit(s.typ, idgen)
field.typ = t
assert t.kind != tyTyped

14
compiler/main.nim
View File

@@ -419,9 +419,14 @@ proc mainCommand*(graph: ModuleGraph) =
# cmdM uses NIF files, not ROD files
graph.config.symbolFiles = disabledSf
setUseIc(true)
# vtable dispatch needs a whole-program vtable layout, which the
# per-module compilation model cannot provide (yet); methods dispatch
# through the classic if-chain dispatchers instead
excl conf.features, Feature.vtables
commandCheck(graph)
of cmdNifC:
setUseIc(true)
excl conf.features, Feature.vtables
# Generate C code from NIF files
wantMainModule(conf)
setOutFile(conf)
@@ -450,7 +455,14 @@ proc mainCommand*(graph: ModuleGraph) =
of cmdUnknown, cmdNone, cmdIdeTools:
rawMessage(conf, errGenerated, "invalid command: " & conf.command)
if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop}:
if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop, cmdM} and
not (conf.cmd == cmdNifC and conf.icBackendStage.len > 0):
# The IC build runs hundreds of internal per-module child processes — the
# frontend `nim m` (cmdM) and the per-module backend stages (cg/emit/merge/
# link). Each would print a `[SuccessX]` summary that is pure noise (and
# misleading: `out: unknownOutput`, or `out: <the whole compiler>` for a
# step that only wrote one `.c.nif`/`.c`). The driving `nim ic` (and koch)
# reports the real result.
if optProfileVM in conf.globalOptions:
echo conf.dump(conf.vmProfileData)
genSuccessX(conf)

28
compiler/mangleutils.nim
View File

@@ -53,7 +53,29 @@ proc mangleParamExt*(s: PSym): string =
result.addInt s.position
proc mangleProcNameExt*(graph: ModuleGraph, s: PSym): string =
result = "__"
# The disambiguator comes first and the module suffix LAST, so the suffix is
# a strippable trailing token: content-addressed cross-module merging chops
# everything from the final `__` to recover a mint-site-independent name.
if s.itemId.isBackendMinted:
# A symbol minted during IC codegen (`idGeneratorForBackend`): its idgen
# starts with an EMPTY per-name disamb table, so its `disamb` restarts at 0
# and collides with same-named sem-time symbols loaded from NIFs (two
# `=destroy` hooks both mangling to `_u2` → "conflicting types for ..." in
# the generated C). These symbols never cross a process boundary (nifc
# lifts, emits and compiles them in one run), so the per-module-unique
# item id is a safe and deterministic discriminator; the `_c` marker keeps
# the namespace disjoint from `_u<disamb>`.
result = "_c"
result.addInt s.itemId.item
else:
result = "_u"
# Use `disamb` rather than `itemId.item`: under incremental compilation a
# symbol loaded from a NIF file gets a fresh, load-order-dependent `itemId.item`
# (from the per-module symbol counter), which is neither stable across the
# processes that compile vs. use a module nor guaranteed distinct from another
# loaded symbol's. `disamb` is assigned deterministically per (module, name)
# and, together with the already-prepended mangled name, yields a unique and
# stable C identifier.
result.addInt s.disamb
result.add "__"
result.add graph.ifaces[s.itemId.module].uniqueName
result.add "_u"
result.addInt s.itemId.item # s.disamb #

369
compiler/modulegraphs.nim
View File

@@ -11,7 +11,7 @@
## represents a complete Nim project. Single modules can either be kept in RAM
## or stored in a rod-file.
import std/[intsets, tables, hashes, strtabs, os, strutils, parseutils]
import std/[intsets, tables, hashes, strtabs, os, strutils, parseutils, sets]
import ../dist/checksums/src/checksums/md5
import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb
@@ -68,6 +68,42 @@ type
enumToStringProcs*: Table[ItemId, PSym]
loadedEnumToStringProcs: Table[string, PSym]
emittedTypeInfo*: Table[string, FileIndex]
instDisambs: Table[(int, int32), ItemId] # (name id, content disamb) ->
# instance, for collision probing in
# `setInstanceDisamb`
icCnifFiles*: seq[string] # `.c.nif` artifacts written by this run
pendingMethodReplays*: seq[PSym] # method registrations loaded under
# `nim nifc`, bucketed only after every
# module is loaded (`flushMethodReplays`)
icImplDeps*: IntSet # NeedsImpl edge tracking under `nim m`:
# module ids (FileIndex) whose routine BODIES
# this compilation consumed at compile time.
# Written to the `.edges` sidecar; deps.nim
# then gates the dependent on those modules'
# IMPL cookie instead of the iface cookie, so
# e.g. `const x = dep.foo()` re-sems when foo's
# body changes. Uniform across body-access
# kinds — the iface cookie hashes signatures
# ONLY (see ast2nif.cookieSd), so every body
# consumer records an edge here: VM-compiled /
# getImpl'ed bodies (recordIcImplDep from vm/
# vmgen), expanded templates (semTemplateExpr)
# and instantiated generics (generateInstance).
# Inline iterators / `inline` procs are NOT
# tracked: they are inlined at codegen, where
# the nifc backend's NIF-mtime invalidation
# already re-codegens their users.
icQualIfaces*: IntSet # module positions whose interface tables were
# populated ONLY for qualified access through a
# module re-export (`import x; export x`); the
# Iface.module stays nil so a later direct
# import still takes the full load path
inVMTransform*: int # >0 while the VM compiles a routine body
# (vmgen.genProc's transformBody): hooks lifted
# there (e.g. for closure-env types of LOADED
# routines) are process-local VM artifacts —
# serializing them would embed references to
# derived env-field syms that no module defines
packageSyms*: TStrTable
deps*: IntSet # the dependency graph or potentially its transitive closure.
@@ -126,6 +162,7 @@ type
procGlobals*: seq[PNode]
nifReplayActions*: Table[int32, seq[PNode]] # module position -> replay actions for NIF
cachedMods: IntSet
hookClosure: IntSet # modules whose serialized hooks were already registered
TPassContext* = object of RootObj # the pass's context
idgen*: IdGenerator
@@ -235,6 +272,18 @@ iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
if s != nil:
yield s
proc reexportedModuleSyms*(g: ModuleGraph; m: PSym): seq[(string, string)] =
## (name, NIF module suffix) of MODULE syms in `m`'s interface — these are
## re-exports (`import x; export x`, added by `reexportSym`) acting as
## qualifiers (`m.x.sym`). Consumed by the NIF writer; semExport does not
## put them into the nkExportStmt children, so the AST walk cannot see them.
result = @[]
var seen = initIntSet()
for s in g.ifaces[m.position].interf.data:
if s != nil and s.kind == skModule and s.position != m.position and
not seen.containsOrIncl(s.position):
result.add (s.name.s, cachedModuleSuffix(g.config, FileIndex s.position))
proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
let importHidden = optImportHidden in m.options
result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name)
@@ -280,21 +329,67 @@ proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym =
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
result = g.loadedOps[op].getOrDefault(key)
#echo "fallback ", key, " ", op, " ", result
when defined(icDbgHash):
if result == nil and op == attachedDestructor:
echo "HOOK MISS key=", key, " table.len=", g.loadedOps[op].len,
" kind=", t.kind, " sym=", (if t.sym != nil: t.sym.name.s else: "NIL")
if key.len > 10:
let probe = key[3 ..< min(key.len, 18)]
for k in g.loadedOps[op].keys:
if probe in k: echo " candidate: ", k
else:
result = nil
proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
## we also need to record this to the packed module.
if not g.attachedOps[op].contains(t.itemId):
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
# Use key-based deduplication for opsLog because different type objects
# (e.g. canon vs orig) can have different itemIds but same structural key
if key notin g.loadedOps[op]:
# Hooks should be written to the module where the type is defined,
# not the module that triggered the registration
let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: module
g.opsLog.add LogEntry(kind: HookEntry, op: op, module: ownerModule, key: key, sym: value)
# Key-based deduplication for opsLog: different type objects (e.g. canon vs
# orig) can have different itemIds but the same structural key.
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
if g.inVMTransform > 0 and g.config.cmd == cmdM:
# hook lifted while the VM compiles a routine body (closure-env types of
# loaded routines): register it for in-process lookup but keep it out of
# the serialized log — it is a process-local artifact whose type graph
# references derived env-field syms that no module's NIF defines
if g.loadedOps[op].getOrDefault(key) == nil:
g.loadedOps[op][key] = value
g.attachedOps[op][t.itemId] = value
return
let existing = g.loadedOps[op].getOrDefault(key)
if existing == nil:
# Stamp the entry with the module whose compilation produced the hook
# (`module`), NOT the type's def module: each `nim m` is a separate
# process, so a hook lifted while compiling a *downstream* module simply
# does not exist in the def module's process — stamping it with the def
# module produced a `LogEntry` that no module ever writes (the def
# module's writer ran in another process that never lifted it; this
# module's writer skips it because `op.module != thisModule`) and codegen
# failed with "'=destroy' operator not found" (e.g. astdef's `TStrTable`,
# whose destroy is first needed by modulegraphs). This holds for nominal
# types as much as for generic/structural instances. Duplicate
# registrations across lifting modules are reconciled deterministically
# at load time (see the HookEntry replay in `replayStateChanges`).
g.opsLog.add LogEntry(kind: HookEntry, op: op, module: module, key: key, sym: value)
g.loadedOps[op][key] = value
elif existing != value:
# Re-registration replacing an earlier sym for the same key. This happens
# legitimately: `createTypeBoundOps` first registers empty `symPrototype`
# placeholders, then `produceSym` replaces them — in particular
# `produceSymDistinctType` replaces a distinct type's placeholder with the
# BASE type's hook (a `distinct string` uses string's `=sink`). The log
# must follow the replacement, otherwise the NIF ships the dead,
# empty-bodied prototype and codegen in another process calls a no-op
# `=sink`/`=copy`, silently losing the value (e.g. `conf.projectPath`
# ended up empty: "cannot open '/'").
g.loadedOps[op][key] = value
var updated = false
for e in mitems(g.opsLog):
if e.kind == HookEntry and e.op == op and e.key == key:
e.sym = value
e.module = module
updated = true
break
if not updated:
g.opsLog.add LogEntry(kind: HookEntry, op: op, module: module, key: key, sym: value)
g.attachedOps[op][t.itemId] = value
proc setAttachedOp*(g: ModuleGraph; module: int; typeId: ItemId; op: TTypeAttachedOp; value: PSym) =
@@ -343,8 +438,10 @@ proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) =
g.enumToStringProcs[t.itemId] = value
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: value.itemId.module.int
g.opsLog.add LogEntry(kind: EnumToStrEntry, module: ownerModule, key: key, sym: value)
# Stamp with the module that owns the generated proc, not the enum's def
# module: the def module's process may never have generated it (same
# "written by nobody" failure as hook entries, see setAttachedOp).
g.opsLog.add LogEntry(kind: EnumToStrEntry, module: value.itemId.module.int, key: key, sym: value)
iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) =
if g.methodsPerGenericType.contains(t.itemId):
@@ -357,6 +454,49 @@ proc addMethodToGeneric*(g: ModuleGraph; module: int; t: PType; col: int; m: PSy
let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: module
g.opsLog.add LogEntry(kind: MethodEntry, module: ownerModule, key: key, sym: m)
proc logMethodDef*(g: ModuleGraph; s: PSym) =
## Log a method registration (`cgmeth.methodDef`) so that importers and
## the backend can rebuild the dispatch buckets (`g.methods`) from the
## NIF replay log — the serialized method ast carries its dispatcher sym
## at `dispatcherPos`, so replay reuses the original dispatcher that all
## call sites reference by name (see `registerLoadedMethod`).
if g.config.cmd in {cmdNifC, cmdM}:
g.opsLog.add LogEntry(kind: MethodEntry, module: s.itemId.module.int,
key: "", sym: s)
proc registerLoadedMethod*(g: ModuleGraph; m: PSym) =
## Rebuild the dispatch buckets from a serialized method registration.
## Buckets group the methods sharing a dispatcher; the dispatcher's BODY
## does not exist in serialized form — `generateIfMethodDispatchers`
## synthesizes it in the backend from the complete bucket.
template dbg(msg: string) =
when defined(icDbgMeth):
echo "[icMeth] replay ", (if m != nil: m.name.s else: "nil"), ": ", msg
if m == nil or sfDispatcher in m.flags: dbg "skip self/nil"; return
if m.ast == nil or dispatcherPos >= m.ast.len:
dbg "no dispatcherPos (len " & $(if m.ast != nil: m.ast.len else: -1) & ")"
return
let dn = m.ast[dispatcherPos]
if dn == nil or dn.kind != nkSym or dn.sym == nil: dbg "empty dispatcher slot"; return
let disp = dn.sym
if sfDispatcher notin disp.flags: dbg "slot sym not a dispatcher"; return
dbg "ok -> bucket of " & disp.name.s & "." & $disp.disamb
for i in 0..<g.methods.len:
if g.methods[i].dispatcher.itemId == disp.itemId:
for existing in g.methods[i].methods:
if existing.itemId == m.itemId: return
g.methods[i].methods.add m
return
g.methods.add (methods: @[m], dispatcher: disp)
proc flushMethodReplays*(g: ModuleGraph) =
## Builds the dispatch buckets from the method registrations collected
## during module loading; called once every module of the program is
## loaded (`nifbackend.generateCode`).
for s in g.pendingMethodReplays:
registerLoadedMethod(g, s)
g.pendingMethodReplays.setLen 0
proc logGenericInstance*(g: ModuleGraph; inst: PSym) =
## Log a generic instance so it gets written to the NIF file.
## This is needed when generic instances are created during compile-time
@@ -365,6 +505,86 @@ proc logGenericInstance*(g: ModuleGraph; inst: PSym) =
let ownerModule = inst.itemId.module.int
g.opsLog.add LogEntry(kind: GenericInstEntry, module: ownerModule, sym: inst)
const
InstanceDisambBit* = 0x4000_0000'i32
## Set in the `disamb` of routine instances whose value is content-derived
## (see `setInstanceDisamb`); keeps them disjoint from the small counter
## range ordinary symbols draw from, so the NIF name `name.disamb.module`
## stays collision-free within a module.
proc setInstanceDisamb*(g: ModuleGraph; inst, generic: PSym;
concreteTypes: openArray[PType]) =
## Under IC, replace a fresh routine instance's counter-based `disamb` with
## a content-derived one: a hash of the generic's identity plus the
## `typeKey` of every concrete type argument — exactly the identity the
## instantiation cache compares. The instance's NIF name
## `name.disamb.modsuffix` then differs only in the module suffix when the
## same instantiation is made by different modules, which is the
## prerequisite for cross-module generic-instance merging (and gives the
## dce analysis its `offers` keys). The hash is computed once, here; it is
## never recomputed — the value travels in the serialized `disamb` field.
if g.config.cmd notin {cmdNifC, cmdM}: return
if isDefined(g.config, "icNoInstKey"): return
var key = generic.name.s
key.add '.'
key.addInt generic.disamb
key.add '.'
key.add modname(generic.itemId.module, g.config)
for t in concreteTypes:
key.add '|'
key.add typeKey(t, g.config, loadTypeCallback, loadSymCallback)
let d = toMD5(key)
var h = (int32(d[0]) or (int32(d[1]) shl 8) or (int32(d[2]) shl 16) or
(int32(d[3] and 0x3F'u8) shl 24)) or InstanceDisambBit
# Same-name hash collisions inside this process get probed to the next
# free value; the loser stays correct (its name keeps the module suffix),
# it merely won't merge cross-module.
while true:
let probe = (inst.name.id, h)
if g.instDisambs.hasKey(probe):
if g.instDisambs[probe] == inst.itemId: break
h = if h == high(int32): InstanceDisambBit else: h + 1
else:
g.instDisambs[probe] = inst.itemId
break
inst.disamb = h
const
HookDisambBit* = 0x2000_0000'i32
## Set in the `disamb` of synthesized type-bound operators and `$enum`
## procs whose value is content-derived (see `setHookDisamb`); disjoint
## from both the small counter range and the `InstanceDisambBit` range.
proc setHookDisamb*(g: ModuleGraph; hook: PSym; opName: string; typ: PType) =
## Under IC, replace a synthesized hook's counter-based `disamb` with a
## content-derived one: a hash of the operation name plus the `typeKey` of
## the type it is bound to. Counter disambs renumber whenever an *earlier*
## hook appears in a re-semmed module, so cached translation units keep
## calling the old `_u<disamb>` C name while the regenerated producer
## defines a new one — the hook flavor of the backend def-migration hole.
## With a content-derived value the hook's NIF name (and hence its C name)
## is stable as long as the type itself is unchanged.
if g.config.cmd notin {cmdNifC, cmdM}: return
if isDefined(g.config, "icNoHookKey"): return
var key = opName
key.add '|'
key.add typeKey(typ, g.config, loadTypeCallback, loadSymCallback)
let d = toMD5(key)
var h = (int32(d[0]) or (int32(d[1]) shl 8) or (int32(d[2]) shl 16) or
(int32(d[3] and 0x1F'u8) shl 24)) or HookDisambBit
# Same-name hash collisions inside this process get probed to the next
# free value (staying below InstanceDisambBit); the loser merely loses
# cross-run name stability.
while true:
let probe = (hook.name.id, h)
if g.instDisambs.hasKey(probe):
if g.instDisambs[probe] == hook.itemId: break
h = if h == InstanceDisambBit - 1'i32: HookDisambBit else: h + 1
else:
g.instDisambs[probe] = hook.itemId
break
hook.disamb = h
proc hasDisabledAsgn*(g: ModuleGraph; t: PType): bool =
let op = getAttachedOp(g, t, attachedAsgn)
result = op != nil and sfError in op.flags
@@ -543,6 +763,7 @@ proc initModuleGraphFields(result: ModuleGraph) =
result.emittedTypeInfo = initTable[string, FileIndex]()
result.cachedFiles = newStringTable()
result.cachedMods = initIntSet()
result.hookClosure = initIntSet()
proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
result = ModuleGraph()
@@ -573,6 +794,15 @@ proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} =
result = true
proc recordIcImplDep*(g: ModuleGraph; s: PSym) =
## NeedsImpl edge tracking, see `icImplDeps`. Called from the compile-time
## body consumption sites (vmgen's proc compilation, the getImpl opcodes).
## Own-module and group-member entries are filtered out when the `.edges`
## sidecar is written.
if g.config.cmd == cmdM and s != nil and s.kind in routineKinds and
s.itemId.module >= 0 and not isBackendMinted(s.itemId):
g.icImplDeps.incl module(s.itemId).int
proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
@@ -658,6 +888,97 @@ proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
assert result != nil
when not defined(nimKochBootstrap):
proc registerLoadedHooks(g: ModuleGraph; logOps: seq[LogEntry]) =
let mainSuffix = getMainModuleSuffix(ast.program)
for x in logOps:
# A dependency's NIF may carry hooks whose syms belong to the module we
# are compiling fresh (e.g. a stale NIF of that very module written by an
# earlier in-process compilation). Loading those would collide with the
# freshly semchecked hook declarations.
if mainSuffix.len > 0 and
cachedModuleSuffix(g.config, x.sym.itemId.module.FileIndex) == mainSuffix:
continue
case x.kind
of HookEntry:
# The same structural hook may be serialized by several instantiating
# modules (a generic/structural instance has no single def site, so each
# using module owns its copy). Pick one deterministic program-wide winner
# by the smaller owning-module name, so every lookup resolves to the same
# sym regardless of module load order.
let existing = g.loadedOps[x.op].getOrDefault(x.key)
if existing == nil or
cachedModuleSuffix(g.config, x.sym.itemId.module.FileIndex) <
cachedModuleSuffix(g.config, existing.itemId.module.FileIndex):
g.loadedOps[x.op][x.key] = x.sym
of EnumToStrEntry:
g.loadedEnumToStringProcs[x.key] = x.sym
of MethodEntry:
# only `methodDef` registrations (empty key) rebuild dispatch
# buckets; the `addMethodToGeneric` flavor (typeKey key) announces
# the uninstantiated generic method, which must never enter a
# bucket (methodsPerGenericType replay is still a todo).
# Under `nim nifc` the replay is deferred: building a bucket forces
# the method's body, and a body loaded mid `loadModuleDependencies`
# registers modules it references in a different path context than
# the lazy loads during codegen do (`flushMethodReplays`).
if x.key.len == 0:
if g.config.cmd == cmdNifC:
g.pendingMethodReplays.add x.sym
else:
registerLoadedMethod(g, x.sym)
else:
discard
proc loadTransitiveHooks(g: ModuleGraph; deps: seq[ModuleSuffix]) =
## Registers the serialized hooks (and enum-to-string procs) of every module
## in the import closure of `deps`. Deliberately does NOT use
## `moduleFromNifFile`: that would register the dep as a fully loaded module
## and a later direct import of it would then skip `replayStateChanges`.
var stack = deps
var interf = initStrTable()
var interfHidden = initStrTable()
while stack.len > 0:
let suffix = stack.pop()
var isKnownFile = false
let fileIdx = g.config.registerNifSuffix(string suffix, isKnownFile)
if not g.hookClosure.containsOrIncl(fileIdx.int):
let precomp = loadNifModule(ast.program, suffix, interf, interfHidden, {})
registerLoadedHooks(g, precomp.logOps)
for d in precomp.deps: stack.add d
proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym =
## A re-exported MODULE (`import x; export x`) acts as a qualifier in the
## re-exporting module's interface (`asmm.x86.nd`). Reconstruct a module
## symbol for it and make its interface tables available for qualified
## lookup (`someSym` reads `g.ifaces[position]`) — WITHOUT registering
## the module: `Iface.module` stays nil so a later direct import still
## takes the full load path (replayStateChanges etc.).
var isKnown = false
let fIdx = g.config.registerNifSuffix(msuffix, isKnown)
if fIdx.int >= g.ifaces.len: setLen(g.ifaces, fIdx.int + 1)
if g.ifaces[fIdx.int].module != nil and
g.ifaces[fIdx.int].module.name.s == mname:
# properly registered already (directly imported earlier): reuse it
return g.ifaces[fIdx.int].module
result = PSym(kindImpl: skModule, itemId: itemId(int32(fIdx), 0'i32),
name: getIdent(g.cache, mname),
infoImpl: newLineInfo(fIdx, 1, 1),
positionImpl: int(fIdx))
setOwner(result, getPackage(g.config, g.cache, fIdx))
if g.ifaces[fIdx.int].module == nil and
not g.icQualIfaces.containsOrIncl(fIdx.int):
var interf = initStrTable()
var interfHidden = initStrTable()
let precomp = loadNifModule(ast.program, ModuleSuffix(msuffix),
interf, interfHidden, {})
# chains: the re-exported module may itself re-export modules
for (n2, s2) in precomp.reexportedModules:
let inner = materializeReexportedModule(g, n2, s2)
if inner != nil:
strTableAdd(interf, inner)
g.ifaces[fIdx.int].interf = interf
g.ifaces[fIdx.int].interfHidden = interfHidden
proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex;
flags: set[LoadFlag] = {}): PrecompiledModule =
## Returns 'nil' if the module needs to be recompiled.
@@ -671,7 +992,7 @@ when not defined(nimKochBootstrap):
let m = PSym(
kindImpl: skModule,
itemId: ItemId(module: int32(fileIdx), item: 0'i32),
itemId: itemId(int32(fileIdx), 0'i32),
name: getIdent(g.cache, splitFile(filename).name),
infoImpl: newLineInfo(fileIdx, 1, 1),
positionImpl: int(fileIdx))
@@ -683,25 +1004,37 @@ when not defined(nimKochBootstrap):
g.ifaces[fileIdx.int].interf,
g.ifaces[fileIdx.int].interfHidden, flags)
result.module = m
for (mname, msuffix) in result.reexportedModules:
let ms = materializeReexportedModule(g, mname, msuffix)
if ms != nil:
strTableAdd(g.ifaces[fileIdx.int].interf, ms)
# Mark module as cached
g.cachedMods.incl fileIdx.int
g.hookClosure.incl fileIdx.int
# Register hooks from NIF index with the module graph
registerLoadedHooks(g, result.logOps)
for x in result.logOps:
case x.kind
of HookEntry:
g.loadedOps[x.op][x.key] = x.sym
of ConverterEntry:
g.ifaces[fileIdx.int].converters.add x.sym
of MethodEntry:
discard "todo"
of EnumToStrEntry:
g.loadedEnumToStringProcs[x.key] = x.sym
discard "dispatch buckets already rebuilt by registerLoadedHooks"
of GenericInstEntry:
raiseAssert "GenericInstEntry should not be in the NIF index"
of HookEntry, EnumToStrEntry:
discard "already done by registerLoadedHooks"
# Register methods per type from NIF index
discard "todo"
# `nim m` loads only its *direct* imports through this proc, but a hook for
# a structural type (e.g. `=destroy` for `seq[PNode]`) lives in the NIF of
# whichever module first lifted it — possibly a dependency of a dependency
# that the current module never imports directly. Walk the whole import
# closure so every serialized hook is visible. (Codegen, `nim nifc`, already
# walks the closure in nifbackend.loadModuleDependencies.)
if g.config.cmd == cmdM:
loadTransitiveHooks(g, result.deps)
proc configComplete*(g: ModuleGraph) =
#rememberStartupConfig(g.startupPackedConfig, g.config)

2
compiler/modules.nim
View File

@@ -32,7 +32,7 @@ proc newModule*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
# We cannot call ``newSym`` here, because we have to circumvent the ID
# mechanism, which we do in order to assign each module a persistent ID.
result = PSym(kindImpl: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32),
result = PSym(kindImpl: skModule, itemId: itemId(int32(fileIdx), 0'i32),
name: getModuleIdent(graph, filename),
infoImpl: newLineInfo(fileIdx, 1, 1))
if not isNimIdentifier(result.name.s):

3
compiler/msgs.nim
View File

@@ -511,6 +511,9 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
## 1-based index (matches editor line numbers); 1st line is for i.line = 1
## last valid line is `numLines` inclusive
if i.fileIndex.int32 < 0: return ""
# line 0 means "unknown": nodes synthesized from an IC-loaded template or
# macro body carry no source position.
if i.line.int < 1: return ""
let num = numLines(conf, i.fileIndex)
# can happen if the error points to EOF:
if i.line.int > num: return ""

479
compiler/nifbackend.nim
View File

@@ -17,18 +17,39 @@
## 1. Compile modules to NIF: nim m mymodule.nim
## 2. Generate C from NIF: nim nifc myproject.nim
import std/[intsets, tables, sets, os]
import std/[intsets, tables, sets, os, algorithm, syncio, times, strutils]
when defined(nimPreviewSlimSystem):
import std/assertions
import ast, options, lineinfos, modulegraphs, cgendata, cgen,
pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif
pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif, typekeys,
cnif
from cgmeth import generateIfMethodDispatchers
import ic / replayer
proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex): seq[PrecompiledModule] =
proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex;
nifFiles: var seq[string];
depFlags: set[LoadFlag] = {LoadFullAst}): seq[PrecompiledModule] =
## Traverse the module dependency graph using a stack.
## Returns all modules that need code generation, in dependency order.
##
## The main module is always loaded with its full AST (it is the codegen
## target). `depFlags` governs the rest: the whole-program backend needs every
## module's full AST (it generates code for all of them), but a per-module
## stage codegens only one target, so it loads the others interface-only
## (`depFlags = {}`) — the interface, hooks, methods and the `(replay ...)`
## directives are loaded regardless of `LoadFullAst`, and demanded bodies are
## fetched lazily from the kept-open stream, so the per-module proc-body ASTs
## (the bulk of the memory) are never materialized for non-targets.
# The main module is loaded by its SOURCE FileIndex, but its serialized
# symbols carry the module's NIF suffix. Pre-alias the suffix to the source
# index so that `registerNifSuffix` does not allocate a second FileIndex for
# the same module, which would split its codegen across two C translation
# units (top-level globals in one, procs in the other → undeclared symbols).
g.config.m.filenameToIndexTbl[cachedModuleSuffix(g.config, mainFileIdx)] = mainFileIdx
let mainModule = moduleFromNifFile(g, mainFileIdx, {LoadFullAst})
nifFiles.add toNifFilename(g.config, mainFileIdx)
var stack: seq[ModuleSuffix] = @[]
result = @[]
@@ -46,9 +67,10 @@ proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex): seq[Precomp
if not visited.containsOrIncl(suffix.string):
var isKnownFile = false
let fileIdx = g.config.registerNifSuffix(suffix.string, isKnownFile)
let precomp = moduleFromNifFile(g, fileIdx, {LoadFullAst})
let precomp = moduleFromNifFile(g, fileIdx, depFlags)
if precomp.module != nil:
result.add precomp
nifFiles.add toNifFilename(g.config, fileIdx)
for dep in precomp.deps:
if not visited.contains(dep.string):
stack.add dep
@@ -62,7 +84,13 @@ proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule =
## Set up a BModule for code generation from a NIF module.
if g.backend == nil:
g.backend = cgendata.newModuleList(g)
result = cgen.newModule(BModuleList(g.backend), module, g.config, idGeneratorFromModule(module))
result = cgen.newModule(BModuleList(g.backend), module, g.config, idGeneratorForBackend(module))
proc isMetaIter(t: PType, closure: RootRef): bool =
# openArray/varargs hooks are sem bookkeeping: no real flow ever demands
# them, and generating one pollutes the TU's type cache with a struct
# descriptor for what must remain a (ptr, len) parameter expansion
t.kind in tyMetaTypes + {tyTyped, tyUntyped, tyNone, tyVarargs, tyOpenArray}
proc finishModule(g: ModuleGraph; bmod: BModule) =
# Finalize the module (this adds it to modulesClosed)
@@ -70,9 +98,52 @@ proc finishModule(g: ModuleGraph; bmod: BModule) =
let initStmt = newNode(nkStmtList)
finalCodegenActions(g, bmod, initStmt)
# Generate dispatcher methods
# NB: the method dispatchers are emitted in `emitMethodDispatchers`,
# between the module loop and this finish loop: their bodies demand the
# method definitions, which can in turn demand definitions from modules
# the backend never loaded — and a TU demand-created during the LAST
# finishModule call would miss `modulesClosed` and never be written.
proc emitMethodDispatchers(g: ModuleGraph) =
## Synthesizes the method dispatcher bodies from the replayed dispatch
## buckets (`registerLoadedMethod`) and emits their definitions into the
## main TU. Main is regenerated on every run, so a dispatcher — whose
## body enumerates the whole program's method set — can never go stale
## inside a cached TU; cross-TU callers prototype it (see genProcLvl3).
let bl = BModuleList(g.backend)
var mainMod: BModule = nil
for m in bl.mods:
if m != nil and m.module != nil and sfMainModule in m.module.flags:
mainMod = m
break
if mainMod == nil: return
generateIfMethodDispatchers(g, mainMod.idgen)
for disp in getDispatchers(g):
genProcLvl3(bmod, disp)
if not containsOrIncl(mainMod.declaredThings, disp.id):
genProcLvl3(mainMod, disp)
proc signatureHasMetaType(t: PType; depth: int = 0): bool =
## Whether a routine signature mentions a compile-time/meta element type
## (`typed`/`untyped` — e.g. `echo`'s `varargs[typed]` — typedesc, static,
## generic param). Such routines are expanded at their call sites and never
## emitted standalone, so the per-module owned-routine seeding must skip them
## (`getTypeDescAux(tyTyped)` otherwise). `tfHasMeta` alone misses the varargs
## element case, hence the explicit scan.
result = false
if t == nil or depth > 8: return false
if t.kind == tyGenericBody:
# The uninstantiated template carried as a `tyGenericInst`'s first child
# always mentions its `tyGenericParam` placeholders, but the instance
# itself is fully concrete (e.g. `var CountTable[SigHash]`). Descending
# here would wrongly flag every routine with a generic-instance parameter
# as meta and drop it from the owned-routine seeding -> undefined symbols
# at link (its only definer never emits it).
return false
if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyStatic, tyGenericParam,
tyAnything, tyFromExpr, tyError}:
return true
for k in t.kids:
if signatureHasMetaType(k, depth + 1): return true
proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
## Generate C code for a single module.
@@ -81,76 +152,366 @@ proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
if bmod == nil:
bmod = setupNifBackendModule(g, precomp.module)
# Apply the module's recorded C compile/link directives (passl/passc/...)
# before generating code: the link step needs them (e.g. math's -lm).
replayBackendActions(g, precomp.module, precomp.topLevel)
# Generate code for the module's top-level statements
if precomp.topLevel != nil:
cgen.genTopLevelStmt(bmod, precomp.topLevel)
proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
## Main entry point for NIF-based C code generation.
## Traverses the module dependency graph and generates C code.
# Per-module backend: emit the bodies of the routines this module OWNS, not
# only the ones its top-level happens to demand. Procs are serialized as lazy
# `(sd ...)` defs (never as `nkProcDef` statements), so `genTopLevelStmt` never
# reaches them; a routine called only from *other* modules would otherwise be
# emitted by nobody, because every module now merely prototypes its foreign
# callees instead of funnelling their bodies (see `cgen.emitsBodyInThisModule`).
# The merge stage's DCE drops whatever turns out globally dead.
if g.config.cmd == cmdNifC and g.config.icBackendStage == "cg":
let modPos = precomp.module.position
for s in moduleSymbolStubs(ast.program, FileIndex modPos):
if s.itemId.module == modPos and
s.kind in {skProc, skFunc, skConverter, skMethod} and
# Only MODULE-level routines: a nested/closure proc (its owner is a
# proc) captures its enclosing scope and cannot be emitted standalone —
# the captured params have no loc → `expr: param not init`. Nested procs
# are emitted via their enclosing routine's lambda-lifting, so seeding
# the enclosing (module-level) routine already covers them.
s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
s.magic == mNone and
# Skip generic instances: they have no single owning-module top-level
# and are emitted by demand (emit-everywhere, deduped by the merge
# stage). An instance has an empty `genericParamsPos` just like a plain
# concrete proc, so only `sfFromGeneric` tells them apart; seeding one
# would force standalone codegen of an instance body whose `when T is X`
# branches were never folded for this path → `genMagicExpr: mIs`.
sfFromGeneric notin s.flags and
# Every other routine the module owns must be emitted here, exported or
# not: a non-exported helper is still reached from another module when a
# `template`/inline routine expands at a call site there (e.g. msgs'
# `internalErrorImpl` behind the `internalError` template), and that
# caller now only prototypes it. `{.error.}`/`compileTime` sentinels and
# bodyless forward decls are not real codegen targets.
{sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
s.typ != nil and not signatureHasMetaType(s.typ) and
s.ast != nil and s.ast.safeLen > bodyPos and
s.ast[genericParamsPos].kind == nkEmpty and
s.ast[bodyPos].kind != nkEmpty:
# a concrete, non-generic, runtime routine with a real body, owned here
requestProcDef(bmod, s)
# Reset backend state
proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
tuple[modules: seq[PrecompiledModule], precompSys: PrecompiledModule,
nifFiles: seq[string]] =
## Shared by the per-module `cg` and `emit` stages: load system + the main
## module's whole import closure and set up a `BModule` for each, so every
## type/symbol resolves and `getCFile` yields the same path both stages use.
## The main module is loaded by its source index (its NIF suffix is aliased to
## it in `loadModuleDependencies`), so it gets exactly one `BModule`.
##
## Only the main module — the codegen target of the stages that use this — is
## loaded with its full AST; every other module is loaded interface-only so
## the whole program's proc bodies are not materialized into this process (that
## was ~1.8 GB for the compiler's main `cg`). The `link` stage codegens nothing
## and only needs each module's `(replay ...)` directives, which load anyway.
resetForBackend(g)
var isKnownFile = false
let systemFileIdx = registerNifSuffix(g.config, "sysma2dyk", isKnownFile)
g.config.m.systemFileIdx = systemFileIdx
#msgs.fileInfoIdx(g.config,
# g.config.libpath / RelativeFile"system.nim")
var precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface})
g.systemModule = precompSys.module
var nifFiles: seq[string] = @[toNifFilename(g.config, systemFileIdx)]
var modules = loadModuleDependencies(g, mainFileIdx, nifFiles, depFlags = {})
# loadModuleDependencies traverses the project's import closure and stops at
# system. The whole-program backend then demand-loads system's own closure
# (locks, allocators, threads, …) during codegen; the per-module backend
# instead makes every one of those a first-class cg/emit target, so load that
# closure here too — otherwise `findTargetModule` cannot resolve their suffix.
block:
var visited = initHashSet[string]()
visited.incl "sysma2dyk"
for m in modules:
visited.incl cachedModuleSuffix(g.config, FileIndex m.module.position)
var stack: seq[ModuleSuffix] = @[]
if precompSys.module != nil:
for dep in precompSys.deps: stack.add dep
while stack.len > 0:
let suffix = stack.pop()
if not visited.containsOrIncl(suffix.string):
var isKnown = false
let fileIdx = registerNifSuffix(g.config, suffix.string, isKnown)
let precomp = moduleFromNifFile(g, fileIdx, {})
if precomp.module != nil:
modules.add precomp
nifFiles.add toNifFilename(g.config, fileIdx)
for dep in precomp.deps: stack.add dep
flushMethodReplays(g)
for m in modules:
discard setupNifBackendModule(g, m.module)
if precompSys.module != nil:
discard setupNifBackendModule(g, precompSys.module)
result = (modules, precompSys, nifFiles)
# Load system module first - it's always needed and contains essential hooks
var precompSys = PrecompiledModule(module: nil)
precompSys = moduleFromNifFile(g, systemFileIdx, {LoadFullAst, AlwaysLoadInterface})
proc loadDepClosure(g: ModuleGraph; targetSuffix: string):
tuple[modules: seq[PrecompiledModule], precompSys: PrecompiledModule,
target: PrecompiledModule] =
## Per-module `cg`/`emit` for a NON-main target: load system + the target
## module + the target's transitive import closure ONLY — not the whole
## program. This is the "process the one file it is passed" model (à la
## Nimony's `hexer c file.nif`): the foreign symbols the target's codegen
## demands are loaded lazily by `ast2nif.moduleId`, which opens any referenced
## module's NIF index on first touch, so a body in a not-loaded module still
## resolves. The closure is loaded as full `BModule`s only so that the
## incidental `g.mods[pos]` accesses during codegen resolve; system's own
## internal closure (allocators, locks, …) is included because a target's
## emit-everywhere codegen can demand those without importing them directly.
##
## The whole program is no longer loaded in this process, which is what bounds
## per-process memory under nifmake's parallel fan-out (the main module's `cg`,
## which still loads everything for NimMain's init list and the method
## dispatchers, runs essentially alone since every other `.c.nif` precedes it).
resetForBackend(g)
var isKnownFile = false
let systemFileIdx = registerNifSuffix(g.config, "sysma2dyk", isKnownFile)
g.config.m.systemFileIdx = systemFileIdx
let precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface})
g.systemModule = precompSys.module
# Load all modules in dependency order using stack traversal
# This must happen BEFORE any code generation so that hooks are loaded into loadedOps
let modules = loadModuleDependencies(g, mainFileIdx)
var modules: seq[PrecompiledModule] = @[]
var visited = initHashSet[string]()
visited.incl "sysma2dyk"
# Only the target is codegen'd, so only it needs its full AST; the closure is
# loaded interface-only (demanded bodies come lazily from the kept-open
# streams), which is what keeps a per-module process light under parallel fan-out.
var isKnown = false
let targetIdx = registerNifSuffix(g.config, targetSuffix, isKnown)
let target = moduleFromNifFile(g, targetIdx, {LoadFullAst})
visited.incl targetSuffix
var stack: seq[ModuleSuffix] = @[]
if target.module != nil:
modules.add target
for dep in target.deps: stack.add dep
if precompSys.module != nil:
for dep in precompSys.deps: stack.add dep
while stack.len > 0:
let suffix = stack.pop()
if not visited.containsOrIncl(suffix.string):
var isKnown2 = false
let fileIdx = registerNifSuffix(g.config, suffix.string, isKnown2)
let precomp = moduleFromNifFile(g, fileIdx, {})
if precomp.module != nil:
modules.add precomp
for dep in precomp.deps: stack.add dep
flushMethodReplays(g)
for m in modules:
discard setupNifBackendModule(g, m.module)
if precompSys.module != nil:
discard setupNifBackendModule(g, precompSys.module)
result = (modules, precompSys, target)
proc findTargetModule(g: ModuleGraph; modules: seq[PrecompiledModule];
precompSys: PrecompiledModule; suffix: string): PrecompiledModule =
## The loaded module whose NIF suffix is `suffix` (the `--icBackendModule`
## value), or a nil module if none matches.
result = PrecompiledModule(module: nil)
for m in modules:
if cachedModuleSuffix(g.config, FileIndex m.module.position) == suffix:
return m
if precompSys.module != nil and
cachedModuleSuffix(g.config, FileIndex precompSys.module.position) == suffix:
return precompSys
proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend codegen (`--icBackendStage:cg --icBackendModule:<suffix>`):
## generate C for the single module named by `icBackendModule` and write only
## its `.c.nif` artifact (no merge, no `.c` render, no cc/link — those are
## separate nifmake rules).
##
## `findPendingModule` routes every demand into the target (emit-everywhere).
##
## A NON-main target loads only its own import closure (`loadDepClosure`); the
## whole program is no longer pulled into every parallel `cg` process. The main
## module still loads everything (`loadBackendModules`) because NimMain's init
## list and the method dispatchers are whole-program; its `cg` runs essentially
## alone (every other `.c.nif` precedes it), so it does not contend for memory.
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
# No whole-program DCE here: each module emits the routines it owns and the
# MERGE stage recomputes the one program-wide live set across all `.c.nif`s.
# Running a whole-program liveness pass over all ~260 NIFs in the main `cg`
# would cost ~900 MB for a result the merge stage throws away.
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
else:
# No whole-program load, hence no whole-program DCE: the target emits its
# full demanded closure and the merge stage drops what is globally dead.
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module codegen: module not found for suffix: " & g.config.icBackendModule)
return
generateCodeForModule(g, target)
let bl = BModuleList(g.backend)
# The main module also owns the whole-program method dispatchers + NimMain.
if sfMainModule in target.module.flags:
emitMethodDispatchers(g)
# NimMain (generated when the main module is finished) must call every other
# module's init/datInit. Those translation units are produced by their own
# `cg` processes, so the calls are registered here from each `.c.nif` meta
# head — which is why the main module's `cg` runs last, after every other
# `.c.nif` exists. Modules without init code (no `.c.nif`) register nothing.
for m in bl.mods:
if m != nil and sfMainModule notin m.module.flags:
let heads = readCnifHeads(getCFile(m).string & ".nif")
registerReusedModuleToMain(bl, m, heads.initRequired, heads.datInitRequired)
let tb = bl.mods[target.module.position]
if tb != nil:
finishModule(g, tb)
# Writes only the target's `.c.nif` (every other loaded module's TU is empty,
# so `cgenWriteModules` emits no artifact for it). cc/link are NOT run here.
cgenWriteModules(g.backend, g.config)
# Always leave a `.c.nif` for the target, even when the module has no code
# (a leaf library whose procs all emit into their users): the per-module
# nifmake graph declares one `.c.nif` output per `cg` rule, so a missing one
# would re-fire the rule forever. An empty artifact renders to an empty `.c`.
if tb != nil:
let artifact = getCFile(tb).string & ".nif"
if not fileExists(artifact):
writeCnifArtifact("", artifact,
semmedNif = toNifFilename(g.config, FileIndex target.module.position),
moduleBase = $getSomeNameForModule(tb))
proc generateMergeStage(g: ModuleGraph) =
## Per-module backend merge (`--icBackendStage:merge`): a pure artifact
## operation, no module graph loaded. Reads every `.c.nif` the `cg` stages
## wrote, computes the global live set and — for each `'u'`-flagged unique
## definition that several `cg` processes emitted (emit-everywhere) — the one
## artifact allowed to embed its body, and writes the decision the `emit`
## stages consume — the cross-process replacement for what used to be
## in-process first-claimant/DCE coordination.
let nimcache = getNimcacheDir(g.config).string
var files: seq[string] = @[]
for artifact in walkFiles(nimcache / "*.c.nif"):
files.add artifact
sort files
let decision = computeMergeDecision(files)
if decision.broken:
rawMessage(g.config, errGenerated,
"per-module backend merge: a .c.nif artifact is missing or unparsable")
return
writeMergeDecision(nimcache / MergeDecisionFile, decision)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icMerge] artifacts: " & $files.len &
" live: " & $decision.live.len & " defs: " & $decision.defs &
" liveDefs: " & $decision.liveDefs & " owned: " & $decision.owners.len
proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend emit (`--icBackendStage:emit --icBackendModule:<suffix>`):
## render the target module's final `.c` from its `.c.nif` and the merge
## decision. Loads the target the same way `cg` does so `getCFile` returns the
## identical path `cg` wrote to (the main module's source-vs-suffix aliasing in
## particular); no codegen runs. A non-main target loads only its own closure
## (`loadDepClosure`) so emit, like `cg`, stays bounded under parallel fan-out.
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
else:
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module emit: module not found for suffix: " & g.config.icBackendModule)
return
let decision = readMergeDecision(getNimcacheDir(g.config).string / MergeDecisionFile)
if decision.broken:
rawMessage(g.config, errGenerated,
"per-module emit: missing or unparsable merge decision " & MergeDecisionFile)
return
let bmod = BModuleList(g.backend).mods[target.module.position]
let cfile = getCFile(bmod).string
let artifact = cfile & ".nif"
var dropped = 0
let code = renderCFromArtifact(artifact, decision, extractFilename(artifact), dropped)
writeFile(cfile, code)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icEmit] " & extractFilename(cfile) & " dropped " &
$dropped & " bodies (" & $code.len & " bytes)"
proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend link (`--icBackendStage:link`): the `emit` stages have
## written every module's `.c`; register them and run the C compiler + linker
## once via `extccomp.callCCompiler` (which parallelizes the per-file cc and
## skips up-to-date objects itself). No codegen runs — the graph is loaded only
## so `getCFile` yields each module's emitted `.c` path.
let (modules, precompSys, _) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
# Set up backend modules for all modules that need code generation
# The per-module `cg` processes each collect their module's C compile/link
# directives (`{.passL: "-lm".}` etc.) via `replayBackendActions`, but those
# live in the cg process and never reach this separate link process. Re-collect
# every loaded module's directives here so the final `callCCompiler` sees them
# (without this, math's `-lm` is lost → undefined `floor`/`pow`/… at link).
for m in modules:
discard setupNifBackendModule(g, m.module)
# Also ensure system module is set up and generated first if it exists
replayBackendActions(g, m.module, m.topLevel)
if precompSys.module != nil:
discard setupNifBackendModule(g, precompSys.module)
generateCodeForModule(g, precompSys)
# Track which modules have been processed to avoid duplicates
var processed = initIntSet()
if precompSys.module != nil:
processed.incl precompSys.module.position
# Generate code for all modules (skip system since it's already processed)
for m in modules:
if not processed.containsOrIncl(m.module.position):
generateCodeForModule(g, m)
# during code generation of `main.nim` we can trigger the code generation
# of symbols in different modules so we need to finish these modules
# here later, after the above loop!
# Important: The main module must be finished LAST so that all other modules
# have registered their init procs before genMainProc uses them.
var mainModule: BModule = nil
for m in BModuleList(g.backend).mods:
replayBackendActions(g, precompSys.module, precompSys.topLevel)
let bl = BModuleList(g.backend)
for m in bl.mods:
if m != nil:
assert m.module != nil
if sfMainModule in m.module.flags:
mainModule = m
else:
finishModule g, m
if mainModule != nil:
finishModule g, mainModule
# Write C files
cgenWriteModules(g.backend, g.config)
# Run C compiler
let cfile = getCFile(m)
# Only modules that are their own cg/emit target produced a `.c`; the rest
# (extra members of system's closure that no build rule targets) had their
# code emit-everywhere'd into the targets, so they have no file to compile.
if not fileExists(cfile.string): continue
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
addFileToCompile(g.config, cf)
if g.config.cmd != cmdTcc:
extccomp.callCCompiler(g.config)
if not g.config.hcrOn:
extccomp.writeJsonBuildInstructions(g.config, g.cachedFiles)
proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
## Main entry point for NIF-based C code generation.
## Traverses the module dependency graph and generates C code.
if g.config.icBackendStage == "cg":
generateCgStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "merge":
generateMergeStage(g)
return
elif g.config.icBackendStage == "emit":
generateEmitStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "link":
generateLinkStage(g, mainFileIdx)
return
else:
rawMessage(g.config, errGenerated,
"the per-module NIF backend requires --icBackendStage:cg|merge|emit|link")

9
compiler/nim.nim
View File

@@ -28,10 +28,12 @@ import
commands, options, msgs, extccomp, main, idents, lineinfos, cmdlinehelper,
pathutils, modulegraphs
from ast2nif import registerNifAstTags
from std/browsers import openDefaultBrowser
from nodejs import findNodeJs
when hasTinyCBackend:
when defined(tinyc): # == hasTinyCBackend; spelled out for the IC dep scanner
import tccgen
when defined(profiler) or defined(memProfiler):
@@ -96,6 +98,11 @@ proc getNimRunExe(conf: ConfigRef): string =
result = ""
proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
# NIF tag registration must not depend on module init order — the IC-built
# compiler orders module init calls differently and the top-level
# `registerTag` initializers then ran against a not-yet-initialized pool,
# corrupting every written NIF (see registerNifAstTags).
registerNifAstTags()
let self = NimProg(
supportsStdinFile: true,
processCmdLine: processCmdLine

8
compiler/nimconf.nim
View File

@@ -11,7 +11,7 @@
import
llstream, commands, msgs, lexer, ast,
options, idents, wordrecg, lineinfos, pathutils, scriptconfig
options, idents, wordrecg, lineinfos, pathutils, scriptconfig, icconfig
import std/[os, strutils, strtabs]
@@ -246,6 +246,12 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile
proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) =
setDefaultLibpath(conf)
# `nim ic` children replay the precompiled config the driver recorded once,
# instead of re-reading the `nim.cfg` chain and re-running `config.nims` in the
# VM. A missing/format-incompatible artifact returns false: fall through to
# normal config loading so an older child or a deleted cache still works.
if conf.icPreparsedConfig.len > 0 and applyIcConfig(conf, conf.icPreparsedConfig):
return
template readConfigFile(path) =
let configPath = path
conf.currentConfigDir = configPath.splitFile.dir.string

66
compiler/options.nim
View File

@@ -29,6 +29,27 @@ const
nimEnableCovariance* = defined(nimEnableCovariance)
icFormatVersion* = "5"
## Version of the IC cache format (the sem-NIF module layout written by
## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
## that layout changes: `commandIc` wipes a nimcache whose `ic.version`
## stamp differs, instead of letting a newer reader mis-parse records
## written by an older compiler (nifmake's rebuild check is mtime-only
## and knows nothing about format changes).
## v2: iface cookie hashes routine SIGNATURES only (no inline-semantics
## body folding); body access now records a NeedsImpl edge instead. A v1
## cache mixes body-sensitive and body-insensitive cookies, so it must be
## wiped rather than warm-rebuilt.
## v3: added the `.s.deps` sidecar (real post-sem imports) and switched the
## macro-generated-import discovery from `icmissing.txt` to it.
## v4: backend C-name scheme change — the module suffix is now the trailing
## token (`name_u<disamb>__<suffix>`, was `name__<suffix>_u<disamb>`), so
## cached `.c.nif` artifacts hold incompatible names and must be wiped.
## v5: data definitions (consts, RTTI) are now wrapped in droppable `'d'`
## cdef directives with an always-present extern declaration, so the
## per-module merge stage can assign them a single owner; old `.c.nif`
## artifacts lack the wrappers.
type # please make sure we have under 32 options
# (improves code efficiency a lot!)
TOption* = enum # **keep binary compatible**
@@ -385,6 +406,44 @@ type
lastCmdTime*: float # when caas is enabled, we measure each command
symbolFiles*: SymbolFilesOption
ic*: bool # whether ic is enabled
icGroup*: HashSet[string] # under `nim m`: absolute paths of the modules in
# this strongly-connected import group. They are all
# compiled from source in one process (so mutual
# recursion resolves in-memory) and each gets its NIF
# written, instead of being loaded from a precompiled
# NIF. See `compiler/deps.nim` (SCC grouping).
icProject*: string # under `nim m`/`nim nifc`: absolute path of the
# ORIGINAL project file. The child's own project file
# is the module being compiled, which would make that
# module's package the "main package" and unfilter
# foreign-package diagnostics; the real project
# restores whole-program filtering semantics.
icPreparsedConfig*: string # under `nim m`/`nim nifc`: path of the precompiled
# config artifact written once by the `nim ic` driver.
# When set, `loadConfigs` replays the recorded
# config-file switches from it instead of re-reading
# the `nim.cfg` chain and re-running `config.nims`
# (which the VM makes expensive) per subprocess.
icConfigSwitches*: seq[tuple[switch, arg: string]]
# the config-file (`passPP`) switches applied while
# loading config, in order. Recorded by every nim
# process; only the `ic` driver serialises them.
# Path-search switches are excluded — the driver
# forwards the resolved `searchPaths` as `--path`.
icBackendStage*: string # under `nim nifc`: which stage of the per-module
# backend this invocation runs — "cg" (codegen one
# module to its `.c.nif`), "merge" (global liveness
# + owner assignment across all `.c.nif`), "emit"
# (render one module's `.c` from its `.c.nif` + the
# merge decision), "link" (cc + link every emitted
# `.c`). Empty = whole-program backend (load all,
# codegen+DCE+cc+link in one process). The stages
# are wired as nifmake rules by `deps.nim`'s backend
# build file. See `compiler/nifbackend.nim`.
icBackendModule*: string # under `nim nifc` with icBackendStage in {cg,emit}:
# the NIF module suffix this invocation codegens or
# emits. The other modules are loaded only so types
# resolve; their definitions are referenced extern.
spellSuggestMax*: int # max number of spelling suggestions for typos
cppDefines*: HashSet[string] # (*)
@@ -431,6 +490,12 @@ type
lastMsgWasDot*: set[StdOrrKind] # the last compiler message was a single '.'
projectMainIdx*: FileIndex # the canonical path id of the main module
projectMainIdx2*: FileIndex # consider merging with projectMainIdx
isMainModule*: bool # `nim m`/IC only: whether the single module being
# semantically checked is the program's real entry point.
# Under IC every module is compiled via `nim m` (which sets
# `sfMainModule` so the module writes its own NIF), so
# `sfMainModule` can no longer answer `isMainModule`. The IC
# build file passes `--isMainModule:on` for the root module.
command*: string # the main command (e.g. cc, check, scan, etc)
commandArgs*: seq[string] # any arguments after the main command
commandLine*: string
@@ -587,6 +652,7 @@ proc newConfigRef*(): ConfigRef =
arcToExpand: newStringTable(modeStyleInsensitive),
m: initMsgConfig(),
cppDefines: initHashSet[string](),
icGroup: initHashSet[string](),
headerFile: "", features: {}, legacyFeatures: {},
configVars: newStringTable(modeStyleInsensitive),
symbols: newStringTable(modeStyleInsensitive),

5
compiler/parser.nim
View File

@@ -54,7 +54,10 @@ import
when not defined(nimCustomAst):
import ast
else:
when defined(nimCustomAst):
# NOTE: explicit negated `when` rather than `else:` — nifler's dep scanner
# guards `when`/`elif` imports with their condition but emits `else:` imports
# unconditionally, which would wrongly schedule this module under `nim ic`.
import plugins / customast
import std/strutils

61
compiler/pipelines.nim
View File

@@ -15,7 +15,7 @@ import ../dist/checksums/src/checksums/sha1
when not defined(leanCompiler):
import jsgen, docgen2
import std/[syncio, objectdollar, assertions, tables, strutils, strtabs]
import std/[syncio, objectdollar, assertions, tables, strutils, strtabs, sets, intsets]
import renderer
import ic/replayer
@@ -243,9 +243,14 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
when not defined(nimKochBootstrap):
# For cmdM: only write NIF for the main module, not for imported modules
# (imported modules should be loaded from existing NIF files)
# (imported modules should be loaded from existing NIF files). Members of the
# current strongly-connected import group (`--icGroup`) are the exception:
# they are compiled from source here, so each must write its own NIF.
let shouldWriteNif = (optCompress in graph.config.globalOptions) or
(graph.config.cmd == cmdM and sfMainModule in module.flags)
(graph.config.cmd == cmdM and
(sfMainModule in module.flags or
(graph.config.icGroup.len > 0 and
toFullPath(graph.config, module.position.FileIndex) in graph.config.icGroup)))
if shouldWriteNif and not graph.config.isDefined("nimscript"):
topLevelStmts.add finalNode
# Collect replay actions from both pragma computations and VM state diff
@@ -259,7 +264,19 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
if m == module:
replayActions.add n
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, replayActions)
# NeedsImpl edge recording: which modules' bodies this process consumed
# at compile time (VM/getImpl). For an --icGroup cycle every member gets
# the union; intra-group entries are filtered by the writer.
var implDeps: seq[int] = @[]
for id in graph.icImplDeps: implDeps.add id
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps, reexportedModuleSyms(graph, module))
# The module's REAL direct imports (incl. macro-generated) for `nim ic`'s
# graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep.
var semDepPaths: seq[string] = @[]
for f in graph.importDeps.getOrDefault(module.position.FileIndex, @[]):
semDepPaths.add toFullPath(graph.config, f)
writeSemDeps(graph.config, module.position.int32, semDepPaths)
result = true
@@ -278,14 +295,34 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
if result == nil:
when not defined(nimKochBootstrap):
# For cmdM: load imports from NIF files (but compile the main module from source)
# Skip when withinSystem is true (compiling system.nim itself)
# Skip when withinSystem is true (compiling system.nim itself).
# Also skip for members of the current strongly-connected import group
# (`--icGroup`): those are mutually recursive with the main module and have
# no precompiled NIF yet, so they must be compiled from source in this same
# process (falling through below) — that resolves the cycle in-memory, the
# same way the non-incremental compiler handles recursive module imports.
if graph.config.cmd == cmdM and
sfMainModule notin flags and
not graph.withinSystem and
not graph.config.isDefined("nimscript"):
not graph.config.isDefined("nimscript") and
(graph.config.icGroup.len == 0 or
toFullPath(graph.config, fileIdx) notin graph.config.icGroup):
let precomp = moduleFromNifFile(graph, fileIdx)
if precomp.module == nil:
let nifPath = toNifFilename(graph.config, fileIdx)
# Macro-generated imports (e.g. chronicles' parseStmt("import
# chronicles/textlines") driven by the chronicles_sinks define) are
# invisible to the static scanner, so this module's NIF was never
# built. The importer already recorded this import via
# addImportFileDep, so flush every module's `.s.deps`: `nim ic` reads
# it, re-derives the graph with the missing node + edge, and reruns
# the frontend. We still error — this process cannot finish sem
# without the import — but the discovery is structured data now, not
# a side-channel file.
for importer, deps in graph.importDeps.pairs:
var paths: seq[string] = @[]
for f in deps: paths.add toFullPath(graph.config, f)
writeSemDeps(graph.config, importer.int32, paths)
globalError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) &
" (expected: " & nifPath & ")")
@@ -364,7 +401,14 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
conf.projectMainIdx2 = projectFile
let packSym = getPackage(graph, projectFile)
var packSym = getPackage(graph, projectFile)
if graph.config.cmd in {cmdM, cmdNifC} and graph.config.icProject.len > 0:
# per-module IC children: the process' project file is the MODULE being
# compiled, which would make its package the "main package" and unfilter
# foreign-package diagnostics (a vendored package's hintAsError promotion
# then aborts builds the whole-program compilation accepts). Use the
# original project, forwarded by deps.nim via --icproject.
packSym = getPackage(graph, fileInfoIdx(graph.config, AbsoluteFile graph.config.icProject))
graph.config.mainPackageId = packSym.getPackageId
graph.importStack.add projectFile
@@ -375,6 +419,9 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
elif graph.config.cmd == cmdM:
# For cmdM: load system.nim from NIF first, then compile the main module
connectPipelineCallbacks(graph)
# Record the main module so the IC loader won't materialise duplicate stubs
# for its own symbols when a dependency (e.g. system) re-exports them.
setIcMainModule(projectFile)
graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
graph.config.libpath / RelativeFile"system.nim")
when not defined(nimKochBootstrap):

2
compiler/pipelineutils.nim
View File

@@ -1,3 +1,4 @@
import std/intsets
import ast, options, lineinfos, pathutils, msgs, modulegraphs, packages
proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} =
@@ -23,4 +24,3 @@ proc prepareConfigNotes*(graph: ModuleGraph; module: PSym) =
proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
result = true
#module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")

19
compiler/semcall.nim
View File

@@ -90,8 +90,14 @@ proc addTypeBoundSymbols(graph: ModuleGraph, arg: PType, name: PIdent,
# argument must be typed first, meaning arguments always
# matching `untyped` are ignored
let t = nominalRoot(arg)
if t != nil and t.owner.kind == skModule:
# search module for routines attachable to `t`
if t != nil and t.owner.kind == skModule and
t.owner.position >= 0 and t.owner.position < graph.ifaces.len:
# search module for routines attachable to `t`.
# Under IC the nominal type may have been loaded from a NIF file, in which
# case its owner module is a stub whose `position` (a NIF-suffix file index)
# has no `ifaces` slot; such type-bound ops are reachable through normal
# imports instead, so skip the direct module scan to avoid an out-of-range
# access.
let module = t.owner
var iter = default(ModuleIter)
var s = initModuleIter(iter, graph, module, name)
@@ -726,6 +732,15 @@ proc indexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode =
result = paramTypesMatch(m, f, a, arg, nil)
if m.genericConverter and result != nil:
instGenericConvertersArg(c, result, m)
when defined(icDbg):
if result == nil and f != nil and a != nil and f.kind == tyEnum:
echo "INDEXMISMATCH f=", typeToString(f), " itemId=", f.itemId,
" uniqueId=", f.uniqueId, " mod=", toFullPath(c.config, f.itemId.module.FileIndex),
" sym=", (if f.sym != nil: $f.sym.itemId else: "nil"), " state=", f.state
let a2 = a.skipTypes({tyRange})
echo " a=", typeToString(a), " itemId=", a2.itemId, " uniqueId=", a2.uniqueId,
" mod=", toFullPath(c.config, a2.itemId.module.FileIndex),
" sym=", (if a2.sym != nil: $a2.sym.itemId else: "nil"), " state=", a2.state
proc inferWithMetatype(c: PContext, formal: PType,
arg: PNode, coerceDistincts = false): PNode =

18
compiler/semdata.nim
View File

@@ -370,7 +370,16 @@ proc addIncludeFileDep*(c: PContext; f: FileIndex) =
discard
proc addImportFileDep*(c: PContext; f: FileIndex) =
discard
# Under `nim m` (the IC frontend) record the REAL direct imports of the
# current module as sem resolves them — including imports a macro generated
# (e.g. chronicles' `parseStmt("import chronicles/textlines")`), which the
# static dependency scanner never sees. `nim ic` writes this set as the
# module's `.s.deps` sidecar and re-derives the build graph from it, so the
# discovery is structured data instead of a build-failure side channel.
if c.config.cmd == cmdM:
let importer = c.module.position.FileIndex
var deps = addr c.graph.importDeps.mgetOrPut(importer, @[])
if f notin deps[]: deps[].add f
proc addPragmaComputation*(c: PContext; n: PNode) =
# Also store for NIF-based IC (cmdM mode or optCompress)
@@ -390,6 +399,13 @@ proc addConverter*(c: PContext, conv: PSym) =
proc addConverterDef*(c: PContext, conv: PSym) =
addConverter(c, conv)
# record the definition for IC: the loader rebuilds Iface.converters from
# the NIF's (repconverter ...) entries (moduleFromNifFile); without the log
# entry a loaded module's converters were invisible to importers and
# implicit conversions silently stopped matching (e.g. faststreams'
# InputStreamHandle -> InputStream at toml_serialization call sites)
c.graph.opsLog.add LogEntry(kind: ConverterEntry, module: c.module.position,
key: "", sym: conv)
proc addPureEnum*(c: PContext, e: PSym) =
assert e != nil

58
compiler/semexprs.nim
View File

@@ -27,6 +27,11 @@ const
proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
flags: TExprFlags = {}; expectedType: PType = nil): PNode =
rememberExpansion(c, n.info, s)
# IC: this expands `s`'s body into the current module's sem, so the module
# depends on that body — record a NeedsImpl (strong) edge to `s`'s module.
# The iface cookie hashes only signatures now, so a template body edit moves
# only the impl cookie, and just the modules that expanded it re-sem.
recordIcImplDep(c.graph, s)
let info = getCallLineInfo(n)
markUsed(c, info, s)
onUse(info, s)
@@ -57,6 +62,16 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
elif {efWantStmt, efAllowStmt} * flags != {}:
result.typ = newTypeS(tyVoid, c)
else:
when defined(icDbgRefc):
echo "[icNoType] semOperand: ", renderTree(result, {renderNoComments}),
" kind=", result.kind,
(if result.kind in {nkCall, nkCommand} and result[0].kind == nkSym:
" calleeTyp=" & (if result[0].sym.typ == nil: "NIL" else:
$result[0].sym.typ.kind & " ret=" &
(if result[0].sym.typ.returnType == nil: "NIL"
else: $result[0].sym.typ.returnType.kind))
else: "")
echo getStackTrace()
localError(c.config, n.info, errExprXHasNoType %
renderTree(result, {renderNoComments}))
result.typ = errorType(c)
@@ -83,6 +98,17 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType
if result.typ == nil and efInTypeof in flags:
result.typ = c.voidType
elif result.typ == nil or result.typ == c.enforceVoidContext:
when defined(icDbgRefc):
echo "[icNoType] semExprWithType: ", renderTree(result, {renderNoComments}),
" kind=", result.kind,
(if result.kind in {nkCall, nkCommand} and result[0].kind == nkSym:
" callee=" & result[0].sym.name.s &
" calleeTyp=" & (if result[0].sym.typ == nil: "NIL" else:
$result[0].sym.typ.kind & " ret=" &
(if result[0].sym.typ.returnType == nil: "NIL"
else: $result[0].sym.typ.returnType.kind))
else: "")
echo getStackTrace()
localError(c.config, n.info, errExprXHasNoType %
renderTree(result, {renderNoComments}))
result.typ = errorType(c)
@@ -199,6 +225,29 @@ proc semOpenSym(c: PContext, n: PNode, flags: TExprFlags, expectedType: PType,
# set symchoice node type back to None
n.typ = newTypeS(tyNone, c)
proc resolveOpenSymDotRhs(c: PContext, n: PNode): PNode =
## Resolves an `nkOpenSym` in the field position of a dot expression.
## The dot handling (`builtinFieldAccess`, `dotTransformation`) matches on
## the node kind of the RHS directly, so the wrapper cannot be left for
## `semExpr` to unwrap; without this the captured symbol degrades to a
## plain identifier that is then only looked up in the instantiation
## context. Mirrors `semOpenSym`: a symbol injected during instantiation
## under the current proc replaces the captured symbol, otherwise the
## captured node is used.
let inner = n[0]
result = inner
if inner.kind != nkSym: return
let id = newIdentNode(inner.sym.name, n.info)
c.isAmbiguous = false
let s2 = qualifiedLookUp(c, id, {})
if s2 != nil and not c.isAmbiguous and s2 != inner.sym:
# only consider symbols defined under the current proc:
var o = s2.owner
while o != nil:
if o == c.p.owner:
return id
o = o.owner
proc semSymChoice(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType = nil): PNode =
if n.kind == nkOpenSymChoice:
result = semOpenSym(c, n, flags, expectedType,
@@ -1526,6 +1575,9 @@ proc builtinFieldAccess(c: PContext; n: PNode; flags: var TExprFlags): PNode =
suggestExpr(c, n)
if exactEquals(c.config.m.trackPos, n[1].info): suggestExprNoCheck(c, n)
if n[1].kind == nkOpenSym:
n[1] = resolveOpenSymDotRhs(c, n[1])
var s = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared, checkModule})
if s != nil:
if s.kind in OverloadableSyms:
@@ -2120,6 +2172,12 @@ proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode =
if c.p.owner.kind notin {skMacro, skTemplate} and
c.p.resultSym != nil and c.p.resultSym.typ.isMetaType:
when defined(icDbgRefc):
echo "[icMetaRet] meta result type for ", c.p.owner.name.s, ": ",
typeToString(c.p.resultSym.typ), " kind=", c.p.resultSym.typ.kind,
" flags=", c.p.resultSym.typ.flags,
" uid=", c.p.resultSym.typ.uniqueId.module, ".", c.p.resultSym.typ.uniqueId.item,
" state=", c.p.resultSym.typ.state
if isEmptyType(result.typ):
# we inferred a 'void' return type:
c.p.resultSym.typ = errorType(c)

13
compiler/semfold.nim
View File

@@ -610,10 +610,21 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
var s = n.sym
case s.kind
of skEnumField:
when defined(icDbg):
if n.typ == nil:
echo "ENUMFIELD niltyp sym=", s.name.s, " symtyp=",
(if s.typ == nil: "nil" else: $s.typ.kind), " lazy=", nfLazyType in n.flags,
" symstate=", s.state, " symid=", s.itemId
result = newIntNodeT(toInt128(s.position), n, idgen, g)
of skConst:
case s.magic
of mIsMainModule: result = newIntNodeT(toInt128(ord(sfMainModule in m.flags)), n, idgen, g)
of mIsMainModule:
# Under `nim m` (IC) `sfMainModule` is set on every module that is being
# compiled (so it writes its own NIF), so it cannot answer `isMainModule`;
# the IC build file marks the real entry point with `--isMainModule:on`.
let isMain = if g.config.cmd == cmdM: g.config.isMainModule
else: sfMainModule in m.flags
result = newIntNodeT(toInt128(ord(isMain)), n, idgen, g)
of mCompileDate: result = newStrNodeT(getDateStr(), n, g)
of mCompileTime: result = newStrNodeT(getClockStr(), n, g)
of mCpuEndian: result = newIntNodeT(toInt128(ord(CPU[g.config.target.targetCPU].endian)), n, idgen, g)

59
compiler/seminst.nim
View File

@@ -119,11 +119,44 @@ proc freshGenSyms(c: PContext; n: PNode, owner, orig: PSym, symMap: var SymMappi
proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind)
proc aliasLoadedTypedescParams(c: PContext, instantiated, orig: PSym): bool =
## When the generic being instantiated had its body LOADED from a NIF (only
## `nim m`/`nim nifc`, only for a generic owned by another module), that body
## re-sems from plain identifiers — ast2nif serialises locals/params as idents,
## not `nkSym`. A `T: typedesc[...]` param referenced as a type must then
## resolve `T` to the bound type, but the instantiated skParam carries the
## concrete type `instantiateProcType` typedesc-skipped it to, which an ident
## lookup cannot use as a type name. Shadow each such param with an `skType`
## alias of the same name in a fresh scope layer (the alias is exactly how Nim
## models "this name denotes a type"). In-process bodies reach the param as
## `nkSym` and never take this path, hence the command gate.
##
## Returns true iff a scope layer was opened; the caller must `closeScope`.
if c.config.cmd notin {cmdM, cmdNifC} or orig == nil or
orig.itemId.module == c.module.position or
orig.typ == nil or orig.typ.n == nil:
return false
result = false
let procParams = instantiated.typ.n
for i in 1..<min(procParams.len, orig.typ.n.len):
if orig.typ.n[i].kind != nkSym: continue
let origParamTyp = orig.typ.n[i].sym.typ
if origParamTyp != nil and origParamTyp.kind == tyTypeDesc and
tfUnresolved in origParamTyp.flags:
if not result:
openScope(c)
result = true
let p = procParams[i].sym
let alias = newSym(skType, p.name, c.idgen, instantiated, p.info)
alias.typ = p.typ
addDecl(c, alias)
proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
if n[bodyPos].kind != nkEmpty:
let procParams = result.typ.n
for i in 1..<procParams.len:
addDecl(c, procParams[i].sym)
let aliasLayer = aliasLoadedTypedescParams(c, result, orig)
maybeAddResult(c, result, result.ast)
inc c.inGenericInst
@@ -152,6 +185,7 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
excl(result, sfForward)
trackProc(c, result, result.ast[bodyPos])
dec c.inGenericInst
if aliasLayer: closeScope(c)
proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
for i in 0..<c.generics.len:
@@ -245,7 +279,7 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
let originalParams = result.n
result.n = originalParams.shallowCopy
for i in 1 ..< originalParams.len:
let resulti = originalParams[i].sym.typ
var resulti = originalParams[i].sym.typ
# twrong_field_caching requires these 'resetIdTable' calls:
if i > FirstParamAt:
resetIdTable(cl.symMap)
@@ -258,6 +292,11 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
let needsStaticSkipping = resulti.kind == tyFromExpr
let needsTypeDescSkipping = resulti.kind == tyTypeDesc and tfUnresolved in resulti.flags
if resulti.kind == tyFromExpr:
if resulti.state == Sealed:
# The generic was loaded from a NIF; do not brand the shared original.
# A tyFromExpr is a placeholder that `replaceTypeVarsT` resolves away,
# so a copy carries no identity that later comparisons could miss.
resulti = copyType(resulti, c.idgen, resulti.owner)
resulti.incl tfNonConstExpr
var paramType = replaceTypeVarsT(cl, resulti)
if needsStaticSkipping:
@@ -276,6 +315,12 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
let param = copySym(oldParam, c.idgen)
setOwner(param, prc)
param.typ = paramType
when defined(icDbgRefc):
echo "[icInst] ", prc.name.s, " param ", oldParam.name.s,
": ", typeToString(resulti), " (kind=", resulti.kind,
" uid=", resulti.uniqueId.module, ".", resulti.uniqueId.item,
" flags=", resulti.flags, ") -> ", typeToString(paramType),
" (kind=", paramType.kind, ")"
# The default value is instantiated and fitted against the final
# concrete param type. We avoid calling `replaceTypeVarsN` on the
@@ -283,6 +328,9 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
if oldParam.ast != nil:
var def = oldParam.ast.copyTree
if def.typ.kind == tyFromExpr:
if def.typ.state == Sealed:
# `copyTree` shares types; see the `resulti` comment above.
def.typ = copyType(def.typ, c.idgen, def.typ.owner)
def.typ.incl tfNonConstExpr
if not isIntLit(def.typ):
def = prepareNode(cl, def)
@@ -374,6 +422,11 @@ proc generateInstance(c: PContext, fn: PSym, pt: LayeredIdTable,
## parameters to their concrete types within the generic instance.
# no need to instantiate generic templates/macros:
internalAssert c.config, fn.kind notin {skMacro, skTemplate}
# IC: instantiating `fn` consumes its generic body in the current module's
# sem — record a NeedsImpl (strong) edge to `fn`'s module. The iface cookie
# hashes only signatures now, so a generic body edit moves only the impl
# cookie, and just the modules that instantiated it re-sem.
recordIcImplDep(c.graph, fn)
# generates an instantiated proc
if c.instCounter > 50:
globalError(c.config, info, "generic instantiation too nested")
@@ -455,6 +508,10 @@ proc generateInstance(c: PContext, fn: PSym, pt: LayeredIdTable,
# This is needed for cyclic module dependencies where generic instances
# may be created in one module but referenced from another.
logGenericInstance(c.graph, result)
# Under IC the instance's NIF name must be canonical across modules:
# derive its `disamb` from the instantiation identity (generic +
# concrete types) instead of the per-module counter.
setInstanceDisamb(c.graph, result, fn, entry.concreteTypes)
# bug #12985 bug #22913
# TODO: use the context of the declaration of generic functions instead
# TODO: consider fixing options as well

8
compiler/semstmts.nim
View File

@@ -2625,6 +2625,14 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
n[genericParamsPos] = proto.ast[genericParamsPos]
n[paramsPos] = proto.ast[paramsPos]
n[pragmasPos] = proto.ast[pragmasPos]
# miscPos holds this definition's *original* generic-param node (kept for
# error messages, see setGenericParamsMisc / issue #1713). For an impl that
# resolves to a forward decl, that node was analysed under the now-discarded
# impl symbol and its generic-param constraint types are owned by it. Adopt
# the prototype's miscPos so the discarded impl sym is fully unreachable —
# otherwise it leaks (via `proto.ast = n` below) as a type owner and gets
# serialized as a phantom duplicate overload under IC.
n[miscPos] = proto.ast[miscPos]
if n[namePos].kind != nkSym: internalError(c.config, n.info, "semProcAux")
n[namePos].sym = proto
if importantComments(c.config) and proto.ast.comment.len > 0:

18
compiler/semtypes.nim
View File

@@ -512,7 +512,13 @@ proc semArrayIndex(c: PContext, n: PNode): PType =
if c.inGenericContext > 0: result.incl tfUnresolved
else:
result = e.typ.skipTypes({tyTypeDesc})
result.incl tfImplicitStatic
if result.state != Sealed:
# For a type loaded from the IC cache we skip the flag instead of
# mutating (or copying) the type: tfImplicitStatic has no readers in
# the compiler, and a copy would get a fresh itemId, breaking enum
# identity (`sameEnumTypes` compares ids) — `arr[enumVal]` on an
# `array[LoadedEnum, T]` would no longer typecheck.
result.incl tfImplicitStatic
elif e.kind in (nkCallKinds + {nkBracketExpr}) and hasUnresolvedArgs(c, e):
if not isOrdinalType(e.typ.skipTypes({tyStatic, tyAlias, tyGenericInst, tySink})):
localError(c.config, n[1].info, errOrdinalTypeExpected % typeToString(e.typ, preferDesc))
@@ -1355,7 +1361,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
for i in 0..<paramType.len - 1:
if paramType[i].kind == tyStatic:
var staticCopy = paramType[i].exactReplica
var staticCopy = paramType[i].exactReplica(c.idgen)
staticCopy.incl tfInferrableStatic
result.rawAddSon staticCopy
else:
@@ -1891,6 +1897,12 @@ proc semTypeExpr(c: PContext, n: PNode; prev: PType): PType =
# by macros. Only macros can summon unnamed types
# and cast spell upon AST. Here we need to give
# it a name taken from left hand side's node
if result.state == Sealed:
# The unnamed type was loaded from a dependency's NIF and must not
# be mutated in place; attach the name to a fresh copy instead.
let orig = result
result = copyType(orig, c.idgen, getCurrOwner(c))
copyTypeProps(c.graph, c.idgen.module, result, orig)
result.sym = prev.sym
result.sym.typ = result
else:
@@ -2178,7 +2190,7 @@ proc semTypeIdent(c: PContext, n: PNode): PSym =
localError(c.config, n.info, errTypeExpected)
return errorSym(c, n)
result = result.typ.sym.copySym(c.idgen)
result.typ = exactReplica(result.typ)
result.typ = exactReplica(result.typ, c.idgen)
result.typ.incl tfUnresolved
if result.kind == skGenericParam:

55
compiler/semtypinst.nim
View File

@@ -272,10 +272,17 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0; expectedType: PT
if n == nil: return
result = copyNode(n)
if n.typ != nil:
if n.typ.kind == tyFromExpr:
var nodeTyp = n.typ
if nodeTyp.kind == tyFromExpr:
# type of node should not be evaluated as a static value
n.typ.incl tfNonConstExpr
result.typ = replaceTypeVarsT(cl, n.typ)
if nodeTyp.state == Sealed:
# IC: do not brand the loaded shared original — a tyFromExpr is a
# placeholder that `replaceTypeVarsT` resolves away, so the copy
# carries no identity later comparisons could miss (mirrors
# `instantiateProcType`)
nodeTyp = copyType(nodeTyp, cl.c.idgen, nodeTyp.owner)
nodeTyp.incl tfNonConstExpr
result.typ = replaceTypeVarsT(cl, nodeTyp)
checkMetaInvariants(cl, result.typ)
case n.kind
of nkNone..pred(nkSym), succ(nkSym)..nkNilLit:
@@ -387,6 +394,13 @@ proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
# don't bind `auto` return type to a previous binding of `auto`
return nil
result = cl.typeMap.lookup(t)
when defined(icDbgRefc):
if t.kind in {tyGenericParam, tyTypeDesc}:
echo "[icBind] lookup ", t.kind, " ", typeToString(t), " uid=", t.uniqueId.module, ".",
t.uniqueId.item, " itemId=", t.itemId.module, ".", t.itemId.item,
" state=", t.state, " flags=", t.flags, " -> ",
(if result != nil: typeToString(result) else: "MISS"),
" allowMeta=", cl.allowMetaTypes
if result == nil:
if cl.allowMetaTypes or tfRetType in t.flags: return
localError(cl.c.config, t.sym.info, "cannot instantiate: '" & typeToString(t) & "'")
@@ -401,7 +415,7 @@ proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
# XXX: relying on allowMetaTypes is a kludge
if cl.allowMetaTypes:
result = t.exactReplica
result = t.exactReplica(cl.c.idgen)
else:
result = copyType(t, cl.c.idgen, t.owner)
copyTypeProps(cl.c.graph, cl.c.idgen.module, result, t)
@@ -446,6 +460,13 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
header[i] = x
propagateToOwner(header, x)
else:
# Under IC `t` may be a loaded dep type (Sealed/immutable); mutating it
# would assert, so propagate into a copy. For non-Sealed types keep
# devel's in-place propagation: unconditionally copying here changes
# `header != t` and with it the cached-instance lookup below, which
# regressed non-IC generic instantiations (arraymancer: a cached
# NimSeqV2 instance with stale flags was returned for a cast target).
if header == t and t.state == Sealed: header = instCopyType(cl, t)
propagateToOwner(header, x)
if header != t:
@@ -497,8 +518,14 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
let bbody = last body
var newbody = replaceTypeVarsT(cl, bbody, isInstValue = true)
cl.skipTypedesc = oldSkipTypedesc
newbody.flags = newbody.flags + (t.flags + body.flags - tfInstClearedFlags)
result.flags = result.flags + newbody.flags - tfInstClearedFlags
let newbodyFlags = newbody.flags + (t.flags + body.flags - tfInstClearedFlags)
if newbody.state != Sealed:
newbody.flags = newbodyFlags
# else: `newbody` is a type loaded from a dep module (it can even be a
# builtin like `int` when the generic's body is computed by a macro) and is
# immutable under IC. Skip the in-place flag accumulation on the shared
# type; the instance `result` still receives the flags below.
result.flags = result.flags + newbodyFlags - tfInstClearedFlags
setToPreviousLayer(cl.typeMap)
@@ -518,8 +545,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
# generics *when the type is constructed*:
cl.c.graph.setAttachedOp(cl.c.module.position, newbody, attachedDeepCopy,
cl.c.instTypeBoundOp(cl.c, dc, result, cl.info, attachedDeepCopy, 1))
if newbody.typeInst == nil:
if newbody.typeInst == nil and newbody.state != Sealed:
# doAssert newbody.typeInst == nil
# An IC-loaded (Sealed) `newbody` keeps whatever `typeInst` its defining
# module serialized; recording this process's first instantiation on the
# shared type is not possible (and was always first-wins anyway).
newbody.typeInst = result
if tfRefsAnonObj in newbody.flags and newbody.kind != tyGenericInst:
# can come here for tyGenericInst too, see tests/metatype/ttypeor.nim
@@ -808,7 +838,11 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType, isInstValue = false):
if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and t.elementType.n != nil:
discard replaceObjBranches(cl, t.elementType.n)
elif result.n != nil and t.kind == tyObject:
elif result.n != nil and t.kind == tyObject and result.state != Sealed:
# A type loaded from the IC cache already had its object branches
# resolved when it was originally compiled, and must not be mutated in
# place (nor copied, which would break object-inheritance identity), so
# only non-Sealed types are processed here.
# Invalidate the type size as we may alter its structure
result.size = -1
result.n = replaceObjBranches(cl, result.n)
@@ -860,7 +894,10 @@ proc recomputeFieldPositions*(t: PType; obj: PNode; currPosition: var int) =
for i in 1..<obj.len:
recomputeFieldPositions(nil, lastSon(obj[i]), currPosition)
of nkSym:
obj.sym.position = currPosition
# A field loaded from the IC cache is already at its final position and must
# not be mutated; only freshly instantiated fields need (re)positioning.
if obj.sym.state != Sealed:
obj.sym.position = currPosition
inc currPosition
else: discard "cannot happen"

61
compiler/sighashes.nim
View File

@@ -52,7 +52,17 @@ proc hashSym(c: var MD5Context, s: PSym) =
c &= ":anon"
else:
var it = s
when defined(icDbgHash):
var ownerSteps = 0
while it != nil:
when defined(icDbgHash):
inc ownerSteps
if ownerSteps >= 1000 and ownerSteps <= 1030:
echo "OWNERLOOP(hashSym) n=", ownerSteps, " sym=", it.name.s, " kind=", it.kind,
" id=", it.itemId, " flags=", it.flags, " state=", it.state,
" start=", s.name.s, " startId=", s.itemId
elif ownerSteps == 1031:
raiseAssert "owner-chain cycle detected, see OWNERLOOP dump above"
c &= it.name.s
c &= "."
it = it.owner
@@ -65,7 +75,17 @@ proc hashTypeSym(c: var MD5Context, s: PSym; conf: ConfigRef) =
else:
var it = s
c &= customPath(conf.toFullPath(s.info))
when defined(icDbgHash):
var ownerSteps = 0
while it != nil:
when defined(icDbgHash):
inc ownerSteps
if ownerSteps >= 1000 and ownerSteps <= 1030:
echo "OWNERLOOP n=", ownerSteps, " sym=", it.name.s, " kind=", it.kind,
" id=", it.itemId, " flags=", it.flags, " state=", it.state,
" start=", s.name.s, " startId=", s.itemId
elif ownerSteps == 1031:
raiseAssert "owner-chain cycle detected, see OWNERLOOP dump above"
if sfFromGeneric in it.flags and it.kind in routineKinds and
it.typ != nil:
hashType c, it.typ, {CoProc}, conf
@@ -102,10 +122,28 @@ proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]; conf: Confi
else:
for i in 0..<n.len: hashTree(c, n[i], flags, conf)
when defined(icDbgHash):
var hashDepth = 0
var hashCalls = 0
var hashMaxDepth = 0
proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: ConfigRef) =
if t == nil:
c &= "\254"
return
when defined(icDbgHash):
inc hashDepth
inc hashCalls
if hashDepth > hashMaxDepth: hashMaxDepth = hashDepth
if hashCalls >= 500_000_000 and hashCalls <= 500_000_300:
echo "HASHLOOP n=", hashCalls, " d=", hashDepth, " kind=", t.kind, " id=", t.itemId,
" uniq=", t.uniqueId, " sym=", (if t.sym != nil: t.sym.name.s else: "NIL"),
" state=", t.state, " owner=", (if t.owner != nil: t.owner.name.s else: "NIL")
elif hashCalls == 500_000_301:
echo "HASHLOOP maxDepth=", hashMaxDepth
raiseAssert "hashType runaway detected, see HASHLOOP dump above"
defer:
dec hashDepth
# Ensure type is fully loaded before hashing to avoid hash changing
# as properties are accessed and trigger lazy loading.
@@ -248,6 +286,29 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi
c.hashType(param.typ, flags, conf)
c &= ','
c.hashType(t.returnType, flags, conf)
elif t.n != nil and t.n.kind == nkFormalParams:
# Under IC a loaded proc type stores its parameters only in `n`; `sons`
# holds just the return type. Hashing `t.signature` would silently drop
# every parameter, collapsing distinct proc types onto one hash, so the
# same logical type got different C struct names in different TUs
# ("incompatible type for argument" on closure args). Hash the return
# type first and then the parameter types from `n` — for from-source
# types `n`'s param types equal `sons[1..]`, so non-IC hashes are
# unchanged. (Same fix as typekeys' tyProc branch.)
c.hashType(t.returnType, flags, conf)
for i in 1..<t.n.len:
let p = t.n[i]
if p.kind == nkSym:
backendEnsureMutable(p.sym)
# The hidden closure env param: under IC, lambda lifting shares the
# routine's AST params with `typ.n`, so the lifted `:envP` leaks into
# the TYPE's params (from-source types never carry it). It is not part
# of the type's identity — `genProcParams` skips it the same way.
if t.callConv == ccClosure and p.sym.name.s == ":envP":
continue
c.hashType(p.sym.typ, flags, conf)
else:
c.hashType(p.typ, flags, conf)
else:
for a in t.signature: c.hashType(a, flags, conf)
c &= char(t.callConv)

13
compiler/sigmatch.nim
View File

@@ -135,6 +135,11 @@ proc put(c: var TCandidate, key, val: PType) {.inline.} =
writeStackTrace()
if c.c.module.name.s == "temp3":
echo "binding ", key, " -> ", val
when defined(icDbgRefc):
if key.kind in {tyGenericParam, tyTypeDesc}:
echo "[icBind] put ", key.kind, " ", typeToString(key), " uid=", key.uniqueId.module, ".",
key.uniqueId.item, " itemId=", key.itemId.module, ".", key.itemId.item,
" state=", key.state, " -> ", typeToString(val)
put(c.bindings, key, val.skipIntLit(c.c.idgen))
proc typeRel*(c: var TCandidate, f, aOrig: PType,
@@ -911,7 +916,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
case typ.kind
of tyStatic:
param = paramSym skConst
param.typ = typ.exactReplica
param.typ = typ.exactReplica(m.c.idgen)
#copyType(typ, c.idgen, typ.owner)
if typ.n == nil:
param.typ.incl tfInferrableStatic
@@ -919,7 +924,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
param.ast = typ.n
of tyFromExpr:
param = paramSym skVar
param.typ = typ.exactReplica
param.typ = typ.exactReplica(m.c.idgen)
#copyType(typ, c.idgen, typ.owner)
else:
param = paramSym skType
@@ -972,7 +977,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
if ff.kind == tyUserTypeClassInst:
result = generateTypeInstance(c, m.bindings, typeClass.sym.info, ff)
else:
result = ff.exactReplica
result = ff.exactReplica(m.c.idgen)
#copyType(ff, c.idgen, ff.owner)
result.n = checkedBody
@@ -2670,7 +2675,7 @@ proc staticAwareTypeRel(m: var TCandidate, f: PType, arg: var PNode): TTypeRelat
# The ast of the type does not point to the symbol.
# Without this we will never resolve a `static proc` with overloads
let copiedNode = copyNode(arg)
copiedNode.typ = exactReplica(copiedNode.typ)
copiedNode.typ = exactReplica(copiedNode.typ, m.c.idgen)
copiedNode.typ.n = arg
arg = copiedNode
typeRel(m, f, arg.typ)

138
compiler/typekeys.nim
View File

@@ -10,7 +10,7 @@
## Based on sighashes.nim but works on astdef directly as we need it in ast2nif.nim.
## Also produces more readable names thanks to treemangler.
import std/assertions
import std/[assertions, sets]
import "../dist/nimony/src/lib" / [treemangler]
import "../dist/nimony/src/gear2" / modnames
@@ -54,6 +54,9 @@ type
m: Mangler
tl: TypeLoader
sl: SymLoader
visited: HashSet[ItemId] # anonymous object types whose fields are currently
# being hashed — a non-mutating guard against endless
# recursion when a field references the type itself.
proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef)
proc symKey(c: var Context; s: PSym; conf: ConfigRef) =
@@ -67,14 +70,26 @@ proc symKey(c: var Context; s: PSym; conf: ConfigRef) =
name.add '.'
name.addInt s.disamb
# The owner may still be an unloaded stub (kind `skStub`): force it in
# before inspecting its kind, otherwise the module suffix is silently
# dropped from the key and def-vs-use keys diverge — e.g. `Lexer`'s base
# class keyed as `TBaseLexer.0.` at nifc vs `TBaseLexer.0.nimqydn3y` at
# sem time, making `getAttachedOp` miss ("'=destroy' operator not found").
template forceLoaded(x: PSym): PSym =
let tmp = x
if tmp != nil and tmp.state == Partial and c.sl != nil: c.sl(tmp)
tmp
let owner = forceLoaded(s.ownerFieldImpl)
let it =
if s.kindImpl == skModule:
s
elif s.kindImpl in skProcKinds and sfFromGeneric in s.flagsImpl and s.ownerFieldImpl.kindImpl != skModule:
s.ownerFieldImpl.ownerFieldImpl
elif s.kindImpl in skProcKinds and sfFromGeneric in s.flagsImpl and
owner != nil and owner.kindImpl != skModule:
forceLoaded(owner.ownerFieldImpl)
else:
s.ownerFieldImpl
if it.kindImpl == skModule:
owner
if it != nil and it.kindImpl == skModule:
name.add '.'
name.add modname(it, conf)
c.m.addSymbol(name)
@@ -138,7 +153,12 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
for a in t.sonsImpl:
c.typeKey a, flags, conf
of tyDistinct:
if CoDistinct in flags:
if t.sonsImpl.len == 0:
# a bare `distinct` typeclass (e.g. `foo(distinct, ...)` matched
# against a `T: type` param) has no base type to key — it IS its kind
withTree c.m, toNifTag(t.kind):
c.m.addEmpty()
elif CoDistinct in flags:
if t.symImpl != nil: symKey(c, t.symImpl, conf)
if t.symImpl == nil or tfFromGeneric in t.flagsImpl:
c.typeKey t.sonsImpl[^1], flags, conf
@@ -147,7 +167,14 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
else:
symKey(c, t.symImpl, conf)
of tyGenericInst:
if sfInfixCall in t.sonsImpl[0].symImpl.flagsImpl:
# The generic head (son[0]) may be a lazily-loaded stub under IC; ensure it
# is materialised before peeking at its symbol. A nil sym means this is not
# an imported C++ generic, so fall through to the normal `skipModifierB`.
var base = t.sonsImpl[0]
if base.state == Partial:
assert c.tl != nil
c.tl(base)
if base.symImpl != nil and sfInfixCall in base.symImpl.flagsImpl:
# This is an imported C++ generic type.
# We cannot trust the `lastSon` to hold a properly populated and unique
# value for each instantiation, so we hash the generic parameters here:
@@ -216,14 +243,48 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
if t.typeInstImpl != nil:
# prevent against infinite recursions here, see bug #8883:
let inst = t.typeInstImpl
if inst.state == Partial:
# a lazily-loaded typeInst stub has no sons until forced in
assert c.tl != nil
c.tl(inst)
t.typeInstImpl = nil # IC: spurious writes are ok since we set it back immediately
assert inst.kind == tyGenericInst
c.typeKey inst.sonsImpl[0], flags, conf
if inst.sonsImpl.len > 0:
c.typeKey inst.sonsImpl[0], flags, conf
for i in 1..<inst.sonsImpl.len-1:
c.typeKey inst.sonsImpl[i], flags, conf
# Match sighashes: generic-instantiation arguments are keyed with
# `CoDistinct` so distinct args are not collapsed to their base.
c.typeKey inst.sonsImpl[i], flags+{CoDistinct}, conf
t.typeInstImpl = inst
elif t.symImpl != nil:
c.symKey(t.symImpl, conf)
# Anonymous / gensym'd object types (e.g. closure environments and
# `ref object` ObjectTypes) share the placeholder name `´anon`, so `symKey`
# alone collapses every one of them onto the same key — which made distinct
# closure-env `=destroy`/`=sink` hooks collide. Mirror sighashes: when the
# type symbol is anonymous/gensym'd, disambiguate further by keying the
# field types and names (or `.empty` when there are none).
template hasFlag(sym: PSym): bool =
{sfAnon, sfGenSym} * sym.flagsImpl != {}
if hasFlag(t.symImpl) or
(t.kind == tyObject and t.ownerFieldImpl != nil and t.ownerFieldImpl.kindImpl == skType and
t.ownerFieldImpl.typImpl != nil and t.ownerFieldImpl.typImpl.kind == tyRef and hasFlag(t.ownerFieldImpl)):
if t.nImpl != nil and t.nImpl.len > 0:
# Guard against endless recursion when a field references this type
# itself. Unlike sighashes (which temporarily clears `sfAnon`/`sfGenSym`
# on the symbol), do NOT mutate: `typeKey` runs during sem — it is
# called unconditionally from `modulegraphs.setAttachedOp` — so a
# mutation that an assertion deeper in `treeKey` left unrestored would
# corrupt the type. `symKey` above already emitted the type's identity,
# so on a back-reference we simply stop.
if not containsOrIncl(c.visited, t.itemId):
c.treeKey(t.nImpl, flags + {CoHashTypeInsideNode}, conf)
c.visited.excl t.itemId
else:
c.m.addIdent "´empty"
# Object inheritance is part of identity: key the base class too.
if t.kind == tyObject and t.sonsImpl.len > 0 and t.sonsImpl[0] != nil:
c.typeKey t.sonsImpl[0], flags, conf
else:
c.m.addIdent "`bug"
of tyFromExpr:
@@ -238,10 +299,19 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
c.symKey(t.nImpl[i].sym, conf)
c.typeKey(t.nImpl[i].sym.typImpl, flags+{CoIgnoreRange}, conf)
else:
for i in 1..<t.sonsImpl.len:
# ALL sons are tuple fields (son 0 included — unlike tyProc, where
# son 0 is the return type). Starting at 1 dropped the first field,
# collapsing e.g. `(PSym, NifIndexEntry)` and `(PType, NifIndexEntry)`
# onto one key, so hook lookup called the wrong `=destroy`/`=sink`
# (incompatible-argument C errors). Mirrors sighashes' `for a in t.kids`.
for i in 0..<t.sonsImpl.len:
c.typeKey t.sonsImpl[i], flags+{CoIgnoreRange}, conf
of tyRange:
if CoIgnoreRange notin flags:
if t.sonsImpl.len == 0:
# bare `range` typeclass: no base type, key the kind alone
withTree c.m, toNifTag(t.kind):
c.m.addEmpty()
elif CoIgnoreRange notin flags:
withTree c.m, toNifTag(t.kind):
c.treeKey(t.nImpl, {}, conf)
c.typeKey(t.sonsImpl[^1], flags, conf)
@@ -254,12 +324,28 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
c.typeKey(t.skipModifierB, flags, conf)
of tyProc:
withTree c.m, (if tfIterator in t.flagsImpl: "itertype" else: "proctype"):
if CoProc in flags and t.nImpl != nil:
# Proc parameter *types* are part of the type's identity. Under IC the
# parameters live in `nImpl` (`sonsImpl` holds only the return type), so a
# loaded proc type has an empty `sonsImpl[1..]`; reading params from there
# would silently drop them and collide every same-return/same-callconv
# closure onto one key (e.g. `proc(cb: proc())` onto bare `proc()`),
# which made hook lookup resolve to the wrong `=copy`. Prefer `nImpl`
# (consistent in-memory and after load); hash param types only, not their
# symbols — parameter names do not affect type identity.
if t.nImpl != nil and t.nImpl.kind == nkFormalParams:
let params = t.nImpl
for i in 1..<params.len:
let param = params[i].sym
c.symKey(param, conf)
c.typeKey(param.typImpl, flags, conf)
if params[i].kind == nkSym:
# The param sym may be a lazily-loaded stub: force it in (as `symKey`
# does) so its type is available, then hash the param *type* only —
# parameter names are not part of the type's identity. Without the
# load the type reads back nil at codegen and the key silently loses
# its parameters (collapsing distinct closure types onto one key).
let ps = params[i].sym
if ps.state == Partial and c.sl != nil: c.sl(ps)
c.typeKey(ps.typImpl, flags, conf)
else:
c.typeKey(params[i].typField, flags, conf)
else:
for i in 1..<t.sonsImpl.len:
c.typeKey(t.sonsImpl[i], flags, conf)
@@ -270,8 +356,12 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
if tfVarargs in t.flagsImpl: c.m.addIdent "´varargs"
of tyArray:
withTree c.m, toNifTag(t.kind):
c.typeKey(t.sonsImpl[^1], flags-{CoIgnoreRange}, conf)
c.typeKey(t.sonsImpl[0], flags-{CoIgnoreRange}, conf)
if t.sonsImpl.len == 0:
# bare `array` typeclass: no element/index types
c.m.addEmpty()
else:
c.typeKey(t.sonsImpl[^1], flags-{CoIgnoreRange}, conf)
c.typeKey(t.sonsImpl[0], flags-{CoIgnoreRange}, conf)
else:
withTree c.m, toNifTag(t.kind):
for i in 0..<t.sonsImpl.len:
@@ -280,6 +370,16 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
c.m.addIdent "´notnil"
proc typeKey*(t: PType; conf: ConfigRef; tl: TypeLoader; sl: SymLoader): string =
var c: Context = Context(m: createMangler(30, -1), tl: tl, sl: sl)
typeKey(c, t, {}, conf)
var c: Context = Context(m: createMangler(30, -1), tl: tl, sl: sl,
visited: initHashSet[ItemId]())
# Mirror the flags liftdestructors uses for its `canonTypes` hash
# (`hashType(skipped, {CoType, CoConsiderOwned, CoDistinct})`): hook keys must
# distinguish what hook *lifting* distinguishes. With empty flags a generic
# `distinct` instance (e.g. nilcheck's `SeqOfDistinct[T, U]`) took the bare
# `symKey` branch — the sym is the generic's and thus SHARED by all
# instances, so `SeqOfDistinct[I, PNode]` and `SeqOfDistinct[I, Nilability]`
# collided onto one key and hook lookup returned the wrong `=sink`
# ("incompatible type for argument" in the generated C). Under `CoDistinct` a
# `tfFromGeneric` distinct keys as sym + base type, keeping instances apart.
typeKey(c, t, {CoType, CoConsiderOwned, CoDistinct}, conf)
result = c.m.extract()

6
compiler/vm.nim
View File

@@ -28,7 +28,7 @@ from magicsys import getSysType
const
traceCode = defined(nimVMDebug)
when hasFFI:
when defined(nimHasLibFFI): # == hasFFI; spelled out for the IC dep scanner
import evalffi
@@ -1310,6 +1310,9 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
var a = regs[rb].node
if a.kind == nkVarTy: a = a[0]
if a.kind == nkSym:
# a macro observed this symbol's implementation: NeedsImpl edge to
# its home module under IC.
recordIcImplDep(c.graph, a.sym)
regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit)
else: copyTree(a.sym.ast)
regs[ra].node.flags.incl nfIsRef
@@ -1319,6 +1322,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
decodeB(rkNode)
let a = regs[rb].node
if a.kind == nkSym:
recordIcImplDep(c.graph, a.sym)
regs[ra].node =
if a.sym.ast.isNil:
newNode(nkNilLit)

10
compiler/vmgen.nim
View File

@@ -36,7 +36,7 @@ import
magicsys, options, lowerings, lineinfos, transf, astmsgs,
treetab
from modulegraphs import getBody
from modulegraphs import getBody, recordIcImplDep
when defined(nimCompilerStacktraceHints):
import std/stackframes
@@ -46,7 +46,7 @@ const
when debugEchoCode:
import std/private/asciitables
when hasFFI:
when defined(nimHasLibFFI): # == hasFFI; spelled out for the IC dep scanner
import evalffi
type
@@ -2467,6 +2467,10 @@ proc optimizeJumps(c: PCtx; start: int) =
proc genProc(c: PCtx; s: PSym): VmProcInfo =
result = c.procToCodePos.getOrDefault(s.id, NoVmProcInfo)
if result.usedRegisters < 0:
# compile-time execution consumes this routine's BODY: under IC that is a
# NeedsImpl dependency on the routine's home module (iface-cookie gating
# alone would miss body-only edits, e.g. `const x = dep.foo()`).
recordIcImplDep(c.graph, s)
#if s.name.s == "outterMacro" or s.name.s == "innerProc":
# echo "GENERATING CODE FOR ", s.name.s
let last = c.code.len-1
@@ -2480,7 +2484,9 @@ proc genProc(c: PCtx; s: PSym): VmProcInfo =
c.procToCodePos[s.id] = result
# thanks to the jmp we can add top level statements easily and also nest
# procs easily:
inc c.graph.inVMTransform
let body = transformBody(c.graph, c.idgen, s, if isCompileTimeProc(s): {} else: {useCache})
dec c.graph.inVMTransform
let procStart = c.xjmp(body, opcJmp, 0)
var p = PProc(blocks: @[], sym: s)
let oldPrc = c.prc

4
compiler/vmops.nim
View File

@@ -36,7 +36,9 @@ from std/osproc import nil
when defined(nimPreviewSlimSystem):
import std/syncio
else:
when not defined(nimPreviewSlimSystem):
# explicit negated `when` rather than `else:` so nifler's dep scanner guards
# this import with its condition (it emits `else:` imports unconditionally).
from std/formatfloat import addFloatRoundtrip, addFloatSprintf

412
doc/ic.md
View File

@@ -2,165 +2,325 @@
Incremental Compilation (IC)
======================================
The ``nim ic`` command provides incremental compilation support for Nim projects,
allowing faster rebuilds by reusing previously compiled intermediate representations
of modules that haven't changed.
The ``nim ic`` command provides incremental compilation for Nim projects. It
decomposes compilation into per-module steps whose results are cached as NIF
files, and uses the external ``nifmake`` build tool to re-run only the steps
whose inputs changed.
This document describes **how `nim ic` works today**, including the edge cases
that shaped the current design. The per-module backend rewrite that earlier
editions of this document listed as a *Plan* has **landed**: the whole-program,
reuse/redirect/def-retention backend is gone and codegen is now a set of
`nifmake`-driven per-module rules (see *The backend*).
Overview
========
Incremental compilation works by decomposing the compilation process into several stages:
The pipeline has two halves driven by one process (`nim ic`, `commandIc` in
``compiler/deps.nim``) that constructs a dependency graph, writes a build file,
and hands it to ``nifmake``:
1. **Parsing** - Source files are parsed into an abstract syntax tree (AST)
2. **Semantic Analysis** - Symbols are resolved and type checking is performed
3. **Code Generation** - Platform-specific code is generated from the analyzed AST
4. **Linking** - The generated code is linked into an executable
1. **Frontend** — per module:
- ``nifler parse --deps`` turns ``.nim`` source into a parsed NIF
(``.p.nif``) plus a static dependency list (``.deps.nif``).
- ``nim m`` (the *semantic* step, `cmdM`) reads the parsed NIF + the
precompiled NIFs of the module's imports, type-checks, and writes the
**semmed NIF** (``.nif``) plus invalidation sidecars (see *Cookies*).
2. **Backend** — ``nim nifc`` (`cmdNifC`, ``compiler/nifbackend.nim``) reads the
semmed NIFs, generates C, compiles and links.
The IC mechanism caches the results of earlier stages in NIF files
(Nim intermediate format): ``.p.nif`` (parsed), ``.deps.nif`` (dependencies),
and ``.nif`` (semantically analyzed). When recompiling, only modules that have
changed need to be reprocessed through the semantic analysis and code generation
stages, significantly reducing compilation time for large projects.
``nifmake`` orders the steps by their input/output files: every `nim m` runs
before the `nim nifc` step that consumes its NIF, and a step re-fires only when
one of its inputs is newer than its outputs. The driver invokes ``nifmake run
--parallel`` by default, so independent steps at the same DAG depth fan out
across cores; pass ``-d:icNoParallel`` to serialize (readable child output when
debugging a build).
NIF File Format
===============
Artifacts (the NIF zoo)
=======================
NIF (Nim Intermediate Format) files are text-based files that use a Lisp-like
syntax. They employ a hybrid format where byte offsets into the text are used for
efficient access, making them simultaneously human-readable and machine-efficient.
The text representation is particularly valuable for debugging and introspection.
Per module ``<suffix>`` (a content hash of the path; see *NIF symbols* below),
under the nimcache directory:
Each ``.nim`` module produces its own ``.nif`` file during compilation.
The NIF format contains:
| File | Producer | Purpose |
| ---- | -------- | ------- |
| ``<s>.p.nif`` | nifler | parsed AST (syntactic) |
| ``<s>.deps.nif`` | nifler | **static** import list (syntactic `import`s) |
| ``<s>.s.deps.nif`` | `nim m` | **real** post-sem imports (incl. macro-generated); see *Discovery* |
| ``<s>.nif`` | `nim m` | semmed module (symbols resolved, typed) |
| ``<s>.iface.nif`` | `nim m` | **iface cookie**: hash of the importer-visible surface |
| ``<s>.impl.nif`` | `nim m` | **impl cookie**: hash of the entire content (bodies included) |
| ``<s>.edges.nif`` | `nim m` | **NeedsImpl edges**: modules whose bodies this sem consumed |
| ``<s>.c.nif`` | `nim nifc` | the C text as a NIF, with def/ref markers for DCE & dedup |
| ``ic_config.cfg.nif`` | driver | precompiled config replayed by every child (`icconfig.nim`) |
| ``ic.version`` | driver | format stamp; a mismatch wipes the cache (`icFormatVersion`) |
- **Header** - Version information (e.g., `(.nif27)`)
- **Dependencies** - List of source files and dependencies
- **Interface** - Exported symbols and their indices
- **Body** - The intermediate representation of the module's code in Lisp-like syntax
The NIF format is designed specifically for Nim and allows efficient serialization
and deserialization of the compiler's intermediate representation while remaining
readable and debuggable by tools and developers.
The ``nim ic`` Switch
=====================
The ``nim ic`` command initiates incremental compilation for a project.
It automatically manages the build process by:
1. Parsing all source files into ``.nif`` format (using the ``nifler`` tool)
2. Performing semantic analysis on modified modules
3. Generating code only for modules with changes or dependencies on changed modules
4. Generating a build file (in NIFMake format) that orchestrates the compilation
5. Executing the build file through ``nifmake``
Prerequisites
-------------
- **nifler** - Tool for parsing Nim source files into NIF format. The ``nim ic`` command uses ``nifler parse --deps`` to generate both parsed files (``.p.nif``) and dependency files (``.deps.nif``).
- **nifmake** - Build orchestration tool that follows dependencies and executes the build rules defined in ``.build.nif`` files.
If these tools are not available, ``nim ic`` will display instructions on how to
obtain them.
Key Modules for IC Logic
NIF symbols and ownership
=========================
The primary modules in the compiler that handle incremental compilation logic are:
(See ``../nifspec/doc/nif-spec.md``.) A global symbol is
``<ident>.<disamb>.<moduleSuffix>``. For a **generic instantiation** the
`<disamb>` is not a counter but a *content hash* — `setInstanceDisamb`
(``modulegraphs.nim``) MD5s the generic's identity plus the `typeKey` of every
concrete type argument, masks it to 30 bits and tags it with `InstanceDisambBit`.
So the only part of the name that varies between two modules making the **same**
instantiation (`seq[Foo]`) is the `<moduleSuffix>`. Two consequences drive the
backend:
- **deps.nim** - Dependency analysis and build file generation. Contains the
``commandIc`` procedure which is the main entry point for the ``nim ic`` command.
This module orchestrates the incremental compilation process, handling dependency
traversal (via ``nifler deps``), build rule generation, and build file creation.
The build file is written to ``nifcache/`` directory. This module also explicitly
models ``system.nim`` as a dependency of all modules.
- **Instance names are content-addressed**: the same instantiation produced in
different modules yields the *same* `<ident>.<disamb>`, so a deterministic dedup
is possible by the *module-suffix-stripped* name. The cross-TU C name
(`ccgtypes.sharedInstanceCName`) and the **merge** stage's live-set/owner
decision (`nifbackend.computeMergeDecision`) both key on this stripped form.
- **The suffix names a mint-site owner.** The `<moduleSuffix>` is the module
*that minted the instance* (the instantiation site), so the same instance has a
different full name in each module that makes it. Because every `cg` process
emits the instances it demands (*emit-everywhere*), the same definition can be
produced by several translation units; the **merge** stage then deterministically
picks the single artifact allowed to embed each body (smallest claimant), which
is the cross-process replacement for the old in-process single-writer machinery.
- **ast2nif.nim** - Core mapping between AST and NIF.
The driver: graph construction (`commandIc`)
============================================
1. Stamp/​wipe the cache by ``icFormatVersion``.
2. Seed the graph with the root module and **`system.nim`**. `system`'s entire
import closure is folded into one node (one `nim m` invocation) — see
*single-writer* below.
3. ``traverseDeps`` runs ``nifler`` per module and reads ``.deps.nif`` to add
import edges.
4. **SCC grouping**: strongly-connected import cycles are collapsed (Tarjan).
A singleton compiles as ``nim m <mod>``; a cycle compiles as one
``nim m <rep> --icGroup:<member>…`` that builds every member *from source* in
one process (resolving the recursion in memory) and writes each member's NIF.
Only edges *leaving* the component become build-graph inputs.
5. **Discovery fixpoint**: write the build file, run ``nifmake``; if it fails,
re-derive the graph from every module's ``.s.deps.nif`` (adding nodes/edges
for imports the static scanner missed), and retry. See *Discovery*.
6. The backend step (`nim nifc`) depends on every module's semmed NIF, so
``nifmake`` runs it last.
**Code, Logic & Debugging**
===========================
Invalidation: the cookie system
================================
This section focuses on the compiler-side code paths, the logic you will
inspect while debugging IC, and a pragmatic manual workflow for bug hunting
using local invocations such as ``nim m --nimcache:nifcache``.
A dependent must re-sem only when a dependency's relevant surface changed. Two
hashes per module (``ast2nif.nim``):
Core places to inspect
- **`compiler/deps.nim`**: generates the NIF-based build file and implements
``commandIc`` (entry point for ``nim ic``). Look for how build rules are
emitted (calls to the NIF builder) and how inputs/outputs are wired.
- **`compiler/modulegraphs.nim`** and **`compiler/pipelines.nim`**:
dependency graph and compilation pipeline integration — useful when a module
is rebuilt unexpectedly.
- **iface cookie** (``.iface.nif``): hashes only the *importer-visible* surface —
exported declarations' **signatures** (for *all* routine kinds: plain procs,
templates, macros, generics, `inline` procs alike), full content for
consts/types, plus import/export/replay/hook records. Routine **bodies are
excluded.** It also chains in the iface cookies of its own dependencies, so a
surface change anywhere in the import closure propagates. A `nim m` rule for a
module depends on its dependencies' iface cookies, so a body-only edit moves no
iface cookie and stops the re-sem cascade.
- **impl cookie** (``.impl.nif``): hashes the *entire* serialized content (private
defs and bodies included), with the module's own iface mixed in.
Understanding the NIF text
- NIF files are human-readable; open the per-module ``.nif`` files in
``nifcache/`` to inspect parsed ASTs, dependency lists and interface tables.
- Because NIF uses textual nodes and byte offsets, tools can quickly seek to
positions in the file — but for debugging you usually only need to read the
file top-to-bottom.
**NeedsImpl edges** (``.edges.nif``): if a module *consumed another module's body*
during sem — a macro expansion, a generic instantiation, a `getImpl`, or a
compile-time call run in the VM — it records a strong edge. The dependent is then
gated on that dependency's **impl** cookie instead of its iface cookie, so e.g.
`const x = dep.foo()` re-sems when `foo`'s body changes. Recording sites:
`semExprs.semTemplateExpr` (templates), `seminst.generateInstance` (generics),
`vmgen.genProc` (VM/macros/CT procs), `vm.opcGetImpl` (`getImpl`). Inline
iterators and `inline` procs are *not* tracked — they are inlined at codegen,
where the backend's NIF-mtime invalidation re-codegens their users.
Manual bug-hunting workflow
- Prepare a clean nimcache directory (relative to your project):
Discovery of macro-generated imports
====================================
```bash
mkdir -p nifcache
```
The static scanner only sees syntactic `import`s. A macro can synthesize one
(chronicles does `parseStmt("import chronicles/textlines")` driven by the
`chronicles_sinks` define). Such an import is invisible until sem runs the macro.
Each `nim m` records the imports it *actually* resolved (via the
``semdata.addImportFileDep`` hook → ``graph.importDeps`` → ``ast2nif.writeSemDeps``)
into ``<s>.s.deps.nif``; a child that fails on a not-yet-built import flushes it
before erroring. The driver re-derives the graph from those sidecars — adding the
missing node + the importer→import edge — and reruns to a fixpoint. (This replaced
an earlier `icmissing.txt` side channel.)
- Parse/semantic-check a single module and write NIF/sem artifacts:
The backend: per-module `nifc` stages
=====================================
```bash
nim m --nimcache:nifcache path/to/module.nim
```
Codegen is no longer one whole-program process. ``nim nifc`` (`cmdNifC`,
``compiler/nifbackend.nim``) is invoked once per **stage** via
``--icBackendStage:<stage>``; `commandIc` emits these as ordinary `nifmake` rules
so "which TUs rebuild" is just "which rules `nifmake` re-fires from input mtimes"
— exactly as the frontend already works. There are four stages:
- ``nim m`` runs the compiler up to the semantic checking stage for the
specified module and emits intermediate cache files into ``nifcache/``.
- Use this to reproduce and isolate failures in the semantic stage.
1. **`cg`** (``--icBackendStage:cg --icBackendModule:<suffix>``) — generate C for
the *single* named module and write only its ``<s>.c.nif`` artifact. A non-main
target loads only its own import closure (`loadDepClosure`), so the whole
program is **not** pulled into every parallel `cg` process. Codegen is still
demand-driven and **emit-everywhere**: a `cg` process emits every entity it
demands (generic instances, hooks, RTTI), referencing nothing `extern`-only.
There is no whole-program DCE here — a liveness pass over all ~260 NIFs would
cost ~900 MB for a result the merge stage recomputes anyway. The **main**
module's `cg` is special: it loads everything (`loadBackendModules`), emits the
whole-program method dispatchers and `NimMain`, and registers every other
module's init/datInit from the `.c.nif` meta heads — so it runs *last*, after
every other ``.c.nif`` exists. Every `cg` rule always leaves a ``.c.nif`` (empty
if the module owns no code) so its nifmake output exists and the rule settles.
2. **`merge`** (``--icBackendStage:merge``) — a pure artifact pass, *no module
graph loaded*. Reads every ``.c.nif``, computes the one program-wide live set
and, for each unique definition that several `cg` processes emitted, the single
artifact allowed to embed its body; writes that to a merge-decision file
(`computeMergeDecision` / `writeMergeDecision`). This is the cross-process
replacement for the old in-process first-claimant + DCE coordination.
3. **`emit`** (``--icBackendStage:emit --icBackendModule:<suffix>``) — render the
target module's final ``.c`` from its ``.c.nif`` and the merge decision
(`renderCFromArtifact`, dropping globally-dead and non-owned bodies). No codegen
runs; the target is loaded only so `getCFile` yields the path `cg` wrote.
4. **`link`** (``--icBackendStage:link``) — register every module's emitted ``.c``
and run `extccomp.callCCompiler` once (it parallelizes per-file cc and skips
up-to-date objects). Per-module C compile/link directives (`{.passL.}` etc.) are
re-collected here via `replayBackendActions`, since the `cg` processes that
originally saw them are separate processes (without this, e.g. `math`'s `-lm`
would be lost → undefined `floor`/`pow` at link).
- Inspect the generated files for that module under ``nifcache/`` (look for
``.nif``, sem/parsed artifacts). Because NIF is text-based you can open and
grep it directly:
Because each stage is a `nifmake` rule keyed on file mtimes, a body-only edit to
one module re-fires that module's `cg`+`emit` (and the `merge`/`link`), not the
whole program — and an unchanged module's `cg` does not run at all.
```bash
sed -n '1,200p' nifcache/ModuleName.nif
grep -n "someSymbol" -n nifcache/ModuleName.nif
```
Edge cases (and why the machinery exists)
=========================================
- To reproduce a full incremental compilation of the project, generate the
build file and run it (``nim ic`` automates this). The build file is generated
in ``nifcache/`` directory. To debug an individual build step, run the command
that the build file would execute manually:
- Parsing step: ``nifler parse --deps input.nim`` (produces ``.p.nif`` and ``.deps.nif``)
- Semantic step: ``nim m --nimcache:nifcache input.nim`` (produces ``.nif``)
- Code generation: ``nim nifc --nimcache:nifcache input.nim`` (produces executable)
- **Single-writer.** Instance type-ids are minted in process-local order, so if
two `nim m` processes both write a module's NIF (e.g. a stdlib module pulled
into `system`'s from-source closure *and* given its own rule), the second
overwrites with different ids and every module checked against the first carries
dangling refs ("symbol has no offset"). Fixed by folding `system`'s closure into
one SCC and by **forwarding the project's defines** to every child so their
`when` bodies (hence import sets and NIF contents) match the scanner's.
- **`when … else: import`.** nifler emits `else`-branch imports unguarded, so a
dead `else: import` would be scheduled. The compiler's own sources were rewritten
to explicit negated `when`s; the vendored nifler later learned to negate prior
conditions for the `else`.
- **`nil` sons of loaded ASTs.** NIF dot-tokens load as `nil` where from-source
ASTs have `nkEmpty`; several passes gained `nil` guards.
- **Sealed loaded types.** Loaded types are `Sealed`; sem/transform mutate via
`unsealForTransform`/`exactReplica(idgen)` (the latter mints a fresh `uniqueId`
so serialized replicas don't collapse).
- **Methods/RTTI ownership.** RTTI and type-bound hooks are emit-everywhere at
`cg` and deduplicated by the `merge` stage, like generic instances; the main
module's `cg` owns the whole-program method dispatchers.
- **Config cost.** Each child re-parsing `nim.cfg` + re-running `config.nims` in
the VM was ~80 ms; replaced by a precompiled `ic_config.cfg.nif` replayed in
`loadConfigs` (`compiler/icconfig.nim`).
- **`koch bootic`** bootstraps the compiler through `nim ic` (a 3-iteration
fixed-point check). It writes its binary to ``bin/nim_ic`` and never clobbers
``bin/nim``.
- Force a cache invalidation for a single module by removing its NIF/sem
artifact and re-running the semantic step:
Resolved by the rewrite
-----------------------
```bash
rm nifcache/ModuleName.nif
nim m --nimcache:nifcache path/to/ModuleName.nim
```
The whole-program backend's hand-rolled mini-`nifmake` — `computeModuleReuse`,
`enforceDefRetention`, `redirectToLiveModule`, the cached-defs/claim bookkeeping
and the standalone `dce.nim` — **is gone**. Reuse is now just per-rule `nifmake`
mtime checks, and the single-writer decision is the `merge` stage. The old
**cross-mm / `--force` `var not init`** hazard dissolved with it: every codegen
rule's config (including `--mm`) is a declared `nifmake` input, so a stale-config
TU is simply rebuilt rather than mixed in. `koch bootic` is green under both `orc`
and `--mm:refc`.
- When investigating incorrect replayed state (pragmas, `{.compile: ...}`):
inspect the replay actions in ``compiler/ic/replayer.nim`` and open the
module's NIF to find the ``toReplay``/action entries that will be executed
during reload.
Known residual hack
-------------------
Tips for efficient debugging
- Use ``--path:...`` flags when invoking ``nim m`` to emulate the exact
search paths used in your project, e.g. ``--path:lib --path:vendor``.
- Compare two successive ``.nif`` files with ``diff`` to see what changed and
why a module was rebuilt.
- `deps.runNifler` still uses `setLastModificationTime` to mark its scan
up-to-date and deletes a stale parsed file to coordinate with the nifmake nifler
rule — the driver duplicating nifmake's freshness logic. It is explicitly
flagged in the source and folds away with a full frontend/nifler split.
Where to change behavior
- Cache invalidation decisions and build-rule emission are implemented in
``compiler/deps.nim``. When investigating surprising
rebuilds, instrument those modules to log the footprint/hash/comparison
outcome.
Status and performance
======================
`nim ic` self-builds the compiler (`koch bootic`'s byte-identical fixed-point
check) under both `orc` and `--mm:refc`, and passes the external-package CI set.
Cold full bootstrap on a 32-core box (`-d:release`, **no edits** — IC's worst
case, since incremental reuse is not exercised):
| | wall | notes |
| - | ---- | ----- |
| `koch boot` (classic) | ~1m00s | reference |
| `koch bootic` (`nim ic`) | ~1m39s | **~1.66×** |
This is down from ~7.5× in the whole-program-backend era. IC does modestly more
aggregate work (more processes, NIF re-parsing of imports per process), but on a
many-core box that overhead is absorbed by the parallel `nim m`/`nifc` fan-out,
and the C compile+link floor is shared with the classic backend. On few-core
machines the cold gap is correspondingly wider — IC trades single-build latency
for incremental latency.
The cold number is the *least* favourable comparison: it pays IC's full per-process
overhead while using none of its incremental machinery. **Warm rebuilds — the
actual point of IC — recompile only the modules whose inputs changed** (a body-only
edit re-fires one module's `cg`+`emit`, not the program), so an edit-driven rebuild
is a small fraction of either full build.
The strategic direction (decided 2026-06-13) is to make this NIF backend
(`cmdNifC`) the **default** code generator. The per-module pipeline above is the
realization of that direction; remaining work is *promotion + deletion* of the
classic path, not new machinery.
Design notes and open decisions
===============================
The per-module backend (above) mirrors Nimony's ``src/nimony/deps.nim``: the
backend stopped re-implementing `nifmake`; each stage is a build rule, so reuse is
just mtime checks and the merge stage is the only cross-module coordination.
Settled vs. open:
- **Ownership.** Emittable entities (generic instances, type-bound hooks, RTTI,
lifted procs) are emit-everywhere at `cg` time and deduplicated at `merge` time
(smallest claimant owns each unique body). The earlier idea of a *static*
per-suffix owner computed before codegen was not needed — content-addressed names
make the merge decision deterministic. The precise owner *rule* (minting module
vs. root-type's module) can still be tuned where it would force a downstream
package to own stdlib code.
- **Remaining cleanup.** The `runNifler` `setLastModificationTime` coordination
(above) folds away with a full frontend/nifler split; dead `when` imports could
also be pruned during the `.s.deps` re-derivation.
Validation bar (held on every change): `koch bootic` must reach its byte-identical
fixed point, and binary size must not regress (DCE parity), across the
external-package CI set.
Code, logic & debugging
========================
Core modules:
- **`compiler/deps.nim`** — graph construction, SCC grouping, discovery fixpoint,
build-file generation; `commandIc`.
- **`compiler/ast2nif.nim`** — AST↔NIF, the cookie hashes (`cookieSd`,
`writeIfaceCookie`, `writeImplCookie`, `writeEdgesFile`, `writeSemDeps`).
- **`compiler/nifbackend.nim`** — the per-module backend stages (`generateCgStage`,
`generateMergeStage`, `generateEmitStage`, `generateLinkStage`).
- **`compiler/cnif.nim`** — `.c.nif` artifact read/write, `computeMergeDecision`,
`renderCFromArtifact`.
- **`compiler/icconfig.nim`** — precompiled config.
- **`compiler/pipelines.nim`** / **`modulegraphs.nim`** — pipeline integration and
the graph state (`importDeps`, `icImplDeps`, `icCnifFiles`, `instDisambs`, …).
Manual workflow:
- Frontend a module: ``nim m --nimcache:nifcache path/to/mod.nim`` (writes
``.nif`` + cookies + ``.s.deps``).
- Backend is stage-based (a bare ``nim nifc main.nim`` errors — there is no
whole-program fallback). The exact per-stage commands `nifmake` runs are in the
``*.backend.build.nif`` build file; rerun one directly against an existing cache,
e.g. ``nim nifc --nimcache:nifcache --icBackendStage:cg --icBackendModule:<suffix> main.nim``
to regenerate one module's ``.c.nif``, then ``--icBackendStage:merge`` /
``:emit`` / ``:link``.
- NIF and ``.c.nif`` files are text — open/grep them directly; ``diff`` two
successive ``.nif`` to see why a module rebuilt.
- Force a re-sem: delete the module's ``.nif`` and rerun `nim m`.
- A stale-cache crash after editing the serialization layout means bumping
``icFormatVersion`` (`compiler/options.nim`).
See also
========
- `nif-spec` - NIF format specification (text format and node grammar):
[nifspec/doc/nif-spec.md](../nifspec/doc/nif-spec.md)
- NIF format spec: [nifspec/doc/nif-spec.md](../nifspec/doc/nif-spec.md)
- NIFC (C-like target) spec: dist/nimony/doc/nifc-spec.md

55
koch.nim
View File

@@ -16,11 +16,11 @@ const
ChecksumsStableCommit = "0b8e46379c5bc1bf73d8b3011908389c60fb9b98" # 2.0.1
SatStableCommit = "e63eaea8baf00bed8bcd5a29ffd8823abb265b39"
NimonyStableCommit = "fca0e938b04695a3aa4e85abcc976571189f2bd2" # unversioned \
NimonyStableCommit = "5fa72628a6867f8ca09f8955a493749cf65f006a" # unversioned \
# Note that Nimony uses Nim as a git submodule but we don't want to install
# Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive
# is **required** here.
# Commit from 2026-06-08
# Commit from 2026-06-14
# examples of possible values for fusion: #head, #ea82b54, 1.2.3
FusionStableHash = "#562467452b32cb7a97410ea177f083e6d8405734"
@@ -76,6 +76,7 @@ Options:
--skipIntegrityCheck skips integrity check when booting the compiler
Possible Commands:
boot [options] bootstraps with given command line options
bootic [options] bootstraps via the incremental compiler (`nim ic`)
distrohelper [bindir] helper for distro packagers
tools builds Nim related tools
toolsNoExternal builds Nim related tools (except external tools,
@@ -406,6 +407,55 @@ proc boot(args: string, skipIntegrityCheck: bool) =
if not skipIntegrityCheck:
echo "[Warning] executables are still not equal"
proc bootic(args: string, skipIntegrityCheck: bool) =
## Like `boot`, but bootstraps the compiler through the NIF-based incremental
## compiler (`nim ic`) instead of `nim c`. Differences from `boot`:
## * It starts from an already-bootstrapped Nim (found via `findStartNim`): the
## csources compiler is far too old to provide the `ic` command, and the
## `-d:nimKochBootstrap` define used by `boot`'s first stage *disables*
## `commandIc`, so neither can be used here.
## * `nim ic` drives the per-module build and the final link itself (via
## `nifmake`), so there is no `--compileOnly` + `jsonscript` split.
## The 3-step fixed-point check is kept: a successful run proves the compiler
## can compile itself under IC and reproduces a stable binary.
var output = "compiler" / "nim".exe
# Deliberately NOT `bin/nim`: `bootic` must not clobber the development
# compiler (that would replace a fast release `bin/nim` with bootic's build
# and slow every later `koch`/`nim` invocation). The IC-bootstrapped binary
# lands at `bin/nim_ic` instead; `bin/nim` is only ever read (via findStartNim).
var finalDest = "bin" / "nim_ic".exe
let smartNimcache = (if "release" in args or "danger" in args: "nimcache/ric_" else: "nimcache/dic_") &
hostOS & "_" & hostCPU
bundleChecksums(false)
let nimStart = findStartNim().quoteShell()
let times = 2 - ord(skipIntegrityCheck)
# `boot` shares the `compiler/nim` output path; remove it so a fully warm
# cache still relinks and iteration 1 cannot adopt a stale foreign binary.
removeFile output
for i in 0..times:
echo "iteration: ", i+1
# Iteration 1 may build incrementally (that's the point of IC), but every
# later iteration must start from a clean cache: with a warm cache a
# no-change rerun correctly rebuilds nothing, so iteration i+1 would just
# keep iteration i's binary and the fixed-point check would be vacuous.
# The check is only meaningful if the freshly built compiler re-translates
# everything.
if i > 0: removeDir smartNimcache
let nimi = if i == 0: nimStart else: i.thVersion
exec "$# ic --nimcache:$# $# compiler" / "nim.nim" %
[nimi, smartNimcache, args]
if sameFileContent(output, i.thVersion):
copyExe(output, finalDest)
echo "executables are equal: SUCCESS! (IC-bootstrapped compiler: ", finalDest, ")"
return
copyExe(output, (i+1).thVersion)
copyExe(output, finalDest)
when not defined(windows):
if not skipIntegrityCheck:
echo "[Warning] executables are still not equal"
# -------------- clean --------------------------------------------------------
const
@@ -744,6 +794,7 @@ when isMainModule:
of cmdArgument:
case normalize(op.key)
of "boot": boot(op.cmdLineRest, skipIntegrityCheck)
of "bootic": bootic(op.cmdLineRest, skipIntegrityCheck)
of "clean": clean(op.cmdLineRest)
of "doc", "docs": buildDocs(op.cmdLineRest & " --d:nimPreviewSlimSystem " & paCode, localDocsOnly, localDocsOut)
of "doc0", "docs0":

5
lib/system/mmdisp.nim
View File

@@ -91,7 +91,10 @@ else:
elif defined(gcMarkAndSweep):
# XXX use 'compileOption' here
include "system/gc_ms"
else:
elif not (defined(nimV2) or usesDestructors):
# equivalent to a plain `else` here, but spelled out so that the IC
# dependency scanner (which sees `else` imports/includes unguarded)
# doesn't schedule system/gc's transitive imports under --mm:orc
include "system/gc"
when not declared(nimNewSeqOfCap) and not defined(nimSeqsV2):

2
tests/codegen/titaniummangle_nim.nim
View File

@@ -1,7 +1,7 @@
discard """
targets: "c"
matrix: "--debugger:native --mangle:nim"
ccodecheck: "'testFunc__titaniummangle95nim_u'"
ccodecheck: "'testFunc_u' \\d+ '__titaniummangle95nim'"
"""
#When debugging this notice that if one check fails, it can be due to any of the above.

18
tests/generics/mopensymdot.nim Normal file
View File

@@ -0,0 +1,18 @@
{.experimental: "openSym".}
import std/hashes
template maxHash(t): untyped = high(t).Hash
template implCaptured() {.dirty.} =
result = maxHash(t)
template implInjected() {.dirty.} =
type Hash = uint8
result = high(t).Hash
proc usesCaptured*[T](t: T): auto =
implCaptured()
proc usesInjected*[T](t: T): auto =
implInjected()

12
tests/generics/topensymdot.nim Normal file
View File

@@ -0,0 +1,12 @@
# the RHS of a dot expression can be wrapped in `nkOpenSym` by the generic
# prepass (e.g. a `x.T` type conversion expanded from a dirty template):
# the captured symbol (`hashes.Hash`, not in scope here) must be used when
# nothing is injected, while a symbol injected during instantiation still
# overrides it
{.experimental: "openSym".}
import mopensymdot
doAssert sizeof(usesCaptured(@[1, 2, 3])) == sizeof(int) # hashes.Hash
doAssert sizeof(usesInjected(@[1, 2, 3])) == 1 # injected uint8

2
tests/js/tcodegendeclproc.nim
View File

@@ -3,7 +3,7 @@ discard """
-1
8
'''
ccodecheck: "'console.log(-1); function fac__tcodegendeclproc_u1(n_p0)'"
ccodecheck: "'console.log(-1); function fac_u' \\d+ '__tcodegendeclproc(n_p0)'"
"""
proc fac(n: int): int {.codegenDecl: "console.log(-1); function $2($3)".} =
return n