IC: progress, hacks included

This commit is contained in:
Araq
2026-06-13 10:08:46 +02:00
parent 002d9ed0ef
commit 7148ae3474
24 changed files with 676 additions and 108 deletions

View File

@@ -77,6 +77,16 @@ proc backendEnsureMutable*(t: PType) {.inline.} =
# ^ IC review this later
if t.state == Partial: loadType(t)
proc unsealForTransform*(t: PType) {.inline.} =
## The transformer/lambda lifting also run inside `nim m` when the VM
## compiles a LOADED routine (macro evaluation, `getImpl`). Their mutations
## are process-local — transformed bodies are never written back to a NIF —
## so downgrade the loaded type to mutable, mirroring the `cmdNifC` loader
## which loads everything `Complete` for exactly this reason (see
## `ast2nif.loadedState`).
if t.state == Partial: loadType(t)
if t.state == Sealed: t.state = Complete
proc owner*(s: PSym): PSym {.inline.} =
if s.state == Partial: loadSym(s)
result = s.ownerFieldImpl
@@ -1133,10 +1143,19 @@ proc copyType*(t: PType, idgen: IdGenerator, owner: PSym): PType =
assignType(result, t)
result.symImpl = t.sym # backend-info should not be copied
proc exactReplica*(t: PType): PType =
proc exactReplica*(t: PType; idgen: IdGenerator): PType =
## Replica that KEEPS `itemId` — the generic-param binding tables
## (`LayeredIdTable`) key on it, so the copy must keep matching its
## original — but mints a FRESH `uniqueId`: uniqueId is the SERIALIZATION
## identity (NIF type names key on it) and must be unique per instance.
## Replicas sharing the original's uniqueId serialized as duplicate defs
## under one NIF name; the loader collapsed them into a single type,
## losing their flag differences (use-site `tfUnresolved` typedescs) or
## their structure (meta instance bodies shadowing a generic's canonical
## body).
result = PType(kind: t.kind, ownerFieldImpl: t.owner, sizeImpl: defaultSize,
alignImpl: defaultAlignment, itemId: t.itemId,
uniqueId: t.uniqueId)
uniqueId: nextTypeId(idgen))
assignType(result, t)
result.symImpl = t.sym # backend-info should not be copied

View File

@@ -26,6 +26,8 @@ import typekeys
import ic / [enum2nif]
proc typeToNifSym(typ: PType; config: ConfigRef): string =
# NOTE: uniqueId is the serialization identity and is unique per instance —
# `exactReplica` keeps only itemId shared with its original (see ast.nim)
assert not typ.uniqueId.isBackendMinted
result = "`t"
result.addInt ord(typ.kind)
@@ -34,6 +36,16 @@ proc typeToNifSym(typ: PType; config: ConfigRef): string =
result.add '.'
result.add modname(typ.uniqueId.module, config)
proc icNifTypeName*(typ: PType; config: ConfigRef): string =
## The serialized NIF name of a type, recorded next to RTTI data
## definitions in the cnif artifact so a later run can re-demand the
## typeinfo when a reused TU still references it (the def-retention
## check). Backend-minted types have no NIF name.
if typ != nil and not typ.uniqueId.isBackendMinted:
result = typeToNifSym(typ, config)
else:
result = ""
proc toHookIndexEntry*(config: ConfigRef; typeId: ItemId; hookSym: PSym): HookIndexEntry =
## Converts a type ItemId and hook symbol to a HookIndexEntry for the NIF index.
let typeSymName = "`t" & $typeId.item & "." & cachedModuleSuffix(config, typeId.module.FileIndex)
@@ -164,6 +176,9 @@ type
#writtenTypes: seq[PType] # types written in this module, to be unloaded later
#writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later
writtenPackages: HashSet[string]
writtenTypeNames: HashSet[string] # NIF names of type defs already emitted;
# `exactReplica` copies share the canonical
# type's uniqueId and thus its NIF name
proc isLocalSym(sym: PSym): bool {.inline.} =
## Every symbol is emitted as a *global* (module-suffixed) name so that its
@@ -299,6 +314,15 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) =
dest.addIntLit typ.alignImpl
dest.addIntLit typ.paddingAtEndImpl
dest.addIntLit typ.itemId.item # nonUniqueId
# `exactReplica` keeps the canonical type's itemId (binding-table key)
# while minting a fresh uniqueId (the NIF name): when the two halves
# name different modules, the loader cannot reconstruct itemId.module
# from the type's name — serialize it explicitly
if typ.itemId.module != typ.uniqueId.module and
not typ.itemId.isBackendMinted:
dest.addStrLit modname(typ.itemId.module, w.infos.config)
else:
dest.addDotToken
writeType(w, dest, typ.typeInstImpl)
#if typ.kind in {tyProc, tyIterator} and typ.nImpl != nil and typ.nImpl.kind != nkFormalParams:
@@ -327,7 +351,17 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) =
# module (or nowhere), leaving dangling references (e.g. `symbol has no
# offset` for a `pointer` type whose itemId.module drifted away).
typ.state = Sealed
writeTypeDef(w, dest, typ)
let name = typeToNifSym(typ, w.infos.config)
if w.writtenTypeNames.containsOrIncl(name):
# BACKSTOP: uniqueId is unique per instance since `exactReplica` mints
# fresh ones, so two defs should never share a NIF name anymore.
# Should an id collision slip through regardless, duplicate defs are
# load-order POISON (the loader keys types by name; results.nim's
# Result body was once shadowed by a meta replica, silently nil-ing
# return types downstream) — degrade to a reference to the first def.
dest.addSymUse pool.syms.getOrIncl(name), NoLineInfo
else:
writeTypeDef(w, dest, typ)
else:
dest.addSymUse pool.syms.getOrIncl(typeToNifSym(typ, w.infos.config)), NoLineInfo
@@ -536,12 +570,14 @@ proc moduleSuffix(conf: ConfigRef; f: FileIndex): string =
proc trImport(w: var Writer; n: PNode) =
for child in n:
if child.kind == nkSym:
if child.kind == nkSym and child.sym.kindImpl == skModule:
# a non-module sym appears for an `import v` inside an unexpanded
# template body (e.g. stew/importops' `when compiles((; import v))`):
# not a dependency edge, the import resolves at the expansion site
w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info)
w.deps.addDotToken # flags
w.deps.addDotToken # type
let s = child.sym
assert s.kindImpl == skModule
let fp = moduleSuffix(w.infos.config, s.positionImpl.FileIndex)
w.deps.addStrLit fp # raw string literal, no wrapper needed
w.deps.addParRi
@@ -579,6 +615,7 @@ var repMethodTag = registerTag("repmethod")
var includeTag = registerTag("include")
var importTag = registerTag("import")
var implTag = registerTag("implementation")
var reexpModTag = registerTag("reexpmod")
proc registerNifAstTags*() =
## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag`
@@ -607,6 +644,7 @@ proc registerNifAstTags*() =
includeTag = registerTag("include")
importTag = registerTag("import")
implTag = registerTag("implementation")
reexpModTag = registerTag("reexpmod")
proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) =
if n == nil:
@@ -686,7 +724,15 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) =
if n[namePos].kind == nkSym:
ast = n[namePos].sym.astImpl
if ast == nil: ast = n
else: skipParams = true
else:
# params can only be recovered from `sym.typ.n` if the routine
# was actually semchecked. A routine nested in a TEMPLATE body
# (e.g. faststreams' `proc consumer(bytesVar: openArray[byte])
# {.gensym.}` inside `consumeOutputs`) has a sym but a nil type —
# its params exist only in the AST; dropping them broke the
# template-param substitution at expansion ("undeclared
# identifier" for the injected name).
skipParams = n[namePos].sym.typImpl != nil
w.withNode dest, ast:
for i in 0 ..< ast.len:
if i == paramsPos and skipParams:
@@ -826,16 +872,20 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) =
# - import/include/export entries, `(replay ...)` macro-cache actions and the
# rep* hook/converter/enumtostr registrations (all eagerly consumed by every
# importer's sem via processTopLevel/loadTransitiveHooks).
# - every EXPORTED `(sd ...)`: full content for consts/types/vars/lets and for
# routines with inline semantics — templates, macros, iterators, generics
# (explicit `(genericparams)` in the routine ast or, for implicitly generic
# procs, `tyGenericParam`-kinded types in the signature) and `inline`-callconv
# procs. Plain procs/funcs/methods/converters hash the signature only, the
# body is skipped. (Nimony hashes generic bodies only via the `.inline` path;
# we close that gap here — generic bodies are instantiated by importers.)
# - every EXPORTED `(sd ...)`: full content for consts/types/vars/lets; for
# EVERY routine kind (plain procs, templates, macros, iterators, generics,
# `inline` procs alike) only the SIGNATURE — the body is skipped. A routine
# body is invisible to a dependent's SEM unless the dependent expands /
# instantiates / VM-runs it, and each of those records a NeedsImpl (strong)
# edge gating the dependent on this module's IMPL cookie instead (see
# `cookieSd`). This keeps the iface cookie body-insensitive, so a body edit
# re-sems only the modules that actually consumed that body — not every
# importer (the old model folded inline-semantics bodies into the iface
# cookie, re-semming all importers on any such body edit).
# - nothing else: private defs and top-level init code are invisible to
# importers' sem (their effects on dependents' CODEGEN are covered by the
# nifc backend's transitive NIF-mtime invalidation, which is unchanged).
# importers' sem (their effects on dependents' CODEGEN — and the codegen
# effect of inline iterator/proc body edits — are covered by the nifc
# backend's transitive NIF-mtime invalidation, which is unchanged).
#
# Token-content hashing only — line infos never enter the hash. Names DEFINED
# inside a hashed (sd) (params, locals, the embedded `(td `tK.item.mod)` defs)
@@ -982,21 +1032,6 @@ proc hashRegion(s: var Sha1State; c: var CookieCtx; buf: TokenBuf;
updateAtom s, t
inc i
proc scanSigTypeMarkers(buf: TokenBuf; start, theEnd: int): bool =
## True if the routine's serialized signature marks it as inline-semantics:
## an `inline` calling convention or a `tyGenericParam`-kinded type
## (implicitly generic proc).
let gpPrefix = "`t" & $ord(tyGenericParam) & "."
var i = start
while i < theEnd:
let t = buf[i]
if t.kind == Ident and pool.strings[t.litId] == "inline":
return true
if t.kind in {Symbol, SymbolDef} and pool.syms[t.symId].startsWith(gpPrefix):
return true
inc i
result = false
proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): int =
## Contributes one `(sd ...)` subtree to the cookie; returns the index past it.
result = nextTree(buf, start)
@@ -1016,20 +1051,18 @@ proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): in
kind = parse(TSymKind, pool.tags[buf[fields[0]].tagId])
var skipFrom = -1
var skipTo = -1
if kind in {skProc, skFunc, skMethod, skConverter}:
var fullBody = scanSigTypeMarkers(buf, fields[7], fields[8])
if kind in routineKinds:
# Routines contribute their SIGNATURE only to the iface cookie. A routine
# body is invisible to a dependent's SEM unless the dependent expands,
# instantiates, or VM-runs it — and each of those records a NeedsImpl
# (strong) edge that gates the dependent on this module's IMPL cookie
# instead (templates -> semTemplateExpr, generics -> generateInstance,
# macros/compile-time procs -> the VM's genProc, getImpl -> opcGetImpl).
# Inline iterators and `inline`-callconv procs are inlined at codegen; the
# nifc backend's transitive NIF-mtime invalidation re-codegens their users.
# So no routine body needs to live in the iface cookie.
let ast = fields[9]
if not fullBody and buf[ast].kind == ParLe:
# routine ast tree: tag flags type name pattern genericParams params ...
var p = ast + 1 # flags atom
p = nextTree(buf, p) # -> type slot
p = nextTree(buf, p) # -> son 0 (name)
p = nextTree(buf, p) # -> son 1 (pattern)
p = nextTree(buf, p) # -> son 2 (genericParams)
if buf[p].kind == ParLe and
pool.tags[buf[p].tagId] == toNifTag(nkGenericParams):
fullBody = true
if not fullBody and buf[ast].kind == ParLe:
if buf[ast].kind == ParLe:
# skip son `bodyPos` (6) of the routine ast tree; NOT the last element —
# sem appends the result sym at `resultPos` (7) after the body.
let astEnd = nextTree(buf, ast)
@@ -1043,8 +1076,8 @@ proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): in
if ok:
skipFrom = p
skipTo = nextTree(buf, p)
# templates/macros/iterators and all non-routine kinds (consts carry their
# value, types their structure incl. default field values): hash everything.
# non-routine kinds (consts carry their value, types their structure incl.
# default field values): hash everything.
hashRegion(s, c, buf, start, result, skipFrom, skipTo, keepFirstDefLiteral = true)
proc scanStmtsForCookie(s: var Sha1State; c: var CookieCtx; buf: TokenBuf) =
@@ -1184,7 +1217,8 @@ proc writeEdgesFile(config: ConfigRef; thisModule: int32; implDeps: seq[int]) =
proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
opsLog: seq[LogEntry];
replayActions: seq[PNode] = @[];
implDeps: seq[int] = @[]) =
implDeps: seq[int] = @[];
reexportedModules: seq[(string, string)] = @[]) =
var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule)
var content = createTokenBuf(300)
@@ -1205,6 +1239,17 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
var bottom = createTokenBuf(300)
w.writeToplevelNode content, bottom, n
# Re-exported MODULES (`import x; export x`): semExport puts only x's
# member syms into the nkExportStmt; the module sym itself reaches the
# exporter's interface via `reexportSym` and acts as a QUALIFIER there
# (`asmm.x86.nd`). Serialize (name, suffix) pairs so the loader can
# rebuild that part of the interface.
for (mname, msuffix) in reexportedModules:
w.deps.addParLe reexpModTag, NoLineInfo
w.deps.addStrLit mname
w.deps.addStrLit msuffix
w.deps.addParRi
# the implTag is used to tell the loader that the
# bottom of the file is the implementation of the module:
content.addParLe implTag, NoLineInfo
@@ -1556,7 +1601,8 @@ proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string;
skip n
result = loadSymStub(c, s, thisModule, localSyms)
else:
raiseAssert "sym expected but got " & $n.kind
raiseAssert "sym expected but got " & $n.kind & (
if n.kind == Ident: " '" & pool.strings[n.litId] & "'" else: "")
proc isStub*(t: PType): bool {.inline.} = t.state == Partial
proc isStub*(s: PSym): bool {.inline.} = s.state == Partial
@@ -1620,6 +1666,13 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms
loadField t.alignImpl
loadField t.paddingAtEndImpl
t.itemId = itemId(t.itemId.module, loadAtom(int32, n)) # nonUniqueId
if n.kind == StringLit:
# itemId.module differs from uniqueId.module (an `exactReplica` of a
# foreign type): restore the canonical module half
t.itemId = itemId(int32(moduleId(c, pool.strings[n.litId])), t.itemId.item)
inc n
elif n.kind == DotToken:
inc n
t.typeInstImpl = loadTypeStub(c, n, localSyms)
t.nImpl = loadNode(c, n, typesModule, localSyms)
@@ -2007,12 +2060,53 @@ proc resolveGlobalSym*(c: var DecodeContext; symAsStr: string): PSym =
if not fileExists(modFile): return nil
result = resolveSym(c, symAsStr, true)
proc resolveGlobalType*(c: var DecodeContext; typeName: string): PType =
## By-name resolution for the backend's def-retention check: a type NIF
## name (see `typeToNifSym`) recorded next to an RTTI definition in a
## `.c.nif` artifact is looked up in the current sem state. Returns nil
## when the type no longer exists — including when its whole module
## vanished from the program (`createTypeStub` asserts on both).
if not typeName.startsWith("`t"): return nil
# `t<kind>.<item>.<suffix>
var i = len("`t")
while i < typeName.len and typeName[i] in {'0'..'9'}: inc i
if i >= typeName.len or typeName[i] != '.': return nil
inc i
while i < typeName.len and typeName[i] in {'0'..'9'}: inc i
if i >= typeName.len or typeName[i] != '.': return nil
let suffix = typeName.substr(i+1)
if suffix.len == 0: return nil
let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string
if not fileExists(modFile): return nil
let module = moduleId(c, suffix)
if c.mods[module].index.getOrDefault(typeName).offset == 0: return nil
result = createTypeStub(c, pool.syms.getOrIncl(typeName))
proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: FileIndex): PSym =
## Tries to resolve a compiler proc from a module by checking the NIF index.
## Returns nil if the symbol doesn't exist.
## Returns nil if the symbol doesn't exist. The NIF disamb is mint order, so
## `name.0.` can be any of the overloads sharing the name — for `newSeq` it
## is the generic magic, not the RTL proc (a refc build then demands codegen
## of the generic and dies on `seq[T]`): enumerate the index entries with
## this basename and pick the one that carries `sfCompilerProc`.
result = nil
let suffix = moduleSuffix(c.infos.config, moduleFileIdx)
let symName = name & ".0." & suffix
result = resolveSym(c, symName, true)
let module = moduleId(c, suffix)
let prefix = name & "."
var candidates: seq[int] = @[]
for key in c.mods[module].index.keys:
if key.len > prefix.len and key.startsWith(prefix):
let sn = parseSymName(key)
if sn.name == name:
candidates.add sn.count
# the loads below can grow `c.mods` (symbols reference other modules), so
# resolve only after the index iteration is done
for count in candidates:
let sym = resolveSym(c, name & "." & $count & "." & suffix, true)
if sym != nil:
loadSym(c, sym)
if sfCompilerProc in sym.flagsImpl:
return sym
proc loadLogOp(c: var DecodeContext; logOps: var seq[LogEntry]; s: var Stream; kind: LogEntryKind; op: TTypeAttachedOp; module: int): PackedToken =
result = next(s)
@@ -2067,6 +2161,8 @@ type
deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of
logOps*: seq[LogEntry]
module*: PSym # set by modulegraphs.nim!
reexportedModules*: seq[(string, string)] # (name, suffix) of re-exported MODULE syms;
# materialized by modulegraphs.nim
proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) =
tok = next(s) # skip `(import`
@@ -2190,6 +2286,24 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag];
t = skipTree(s)
elif t.tagId == importTag:
loadImport(c, s, result.deps, t)
elif t.tagId == reexpModTag:
# a re-exported MODULE: (reexpmod "name" "suffix"); the module sym
# is a qualifier in this module's interface — materialized by the
# caller (modulegraphs), which can register interface tables
t = next(s)
var mname = ""
var msuffix = ""
if t.kind == StringLit:
mname = pool.strings[t.litId]
t = next(s)
if t.kind == StringLit:
msuffix = pool.strings[t.litId]
t = next(s)
if t.kind != ParRi:
raiseAssert "expected ParRi in reexpmod entry of module " & suffix
t = next(s)
if mname.len > 0 and msuffix.len > 0:
result.reexportedModules.add (mname, msuffix)
elif t.tagId == implTag:
cont = false
elif LoadFullAst in flags:

View File

@@ -394,7 +394,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; n
# variable. Thus, we create a temporary pointer variable instead.
let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray
if needsIndirect:
n.typ = n.typ.exactReplica
n.typ = n.typ.exactReplica(p.module.idgen)
n.typ.incl tfVarIsPtr
a = initLocExprSingleUse(p, n)
a = withTmpIfNeeded(p, a, needsTmp)

View File

@@ -416,6 +416,12 @@ proc getSimpleTypeDesc(m: BModule; typ: PType): Rope =
m.typeCache[sig] = result
proc pushType(m: BModule; typ: PType) =
when defined(icDbgRefc):
if typ.kind == tySequence and
typ.elementType.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyGenericParam:
echo "[icRefc] pushType seq-of-genericparam t=", typeToString(typ),
" itemId=", typ.itemId.module, ".", typ.itemId.item, " mod=", m.module.name.s
echo getStackTrace()
for i in 0..high(m.typeStack):
# pointer equality is good enough here:
if m.typeStack[i] == typ: return
@@ -1164,6 +1170,11 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes
tyUserTypeClass, tyUserTypeClassInst, tyInferred:
result = getTypeDescAux(m, skipModifier(t), check, kind)
else:
when defined(icDbgRefc):
echo "[icRefc] getTypeDescAux ", t.kind, " t=", typeToString(t),
" origTyp=", typeToString(origTyp), " t.itemId=", t.itemId.module, ".", t.itemId.item,
" sym=", (if t.sym != nil: t.sym.name.s else: "nil"),
" owner=", (if t.owner != nil: t.owner.name.s else: "nil")
internalError(m.config, "getTypeDescAux(" & $t.kind & ')')
result = ""
# fixes bug #145:
@@ -1202,6 +1213,10 @@ proc finishTypeDescriptions(m: BModule) =
var check = initIntSet()
while i < m.typeStack.len:
let t = m.typeStack[i]
when defined(icDbgRefc):
echo "[icRefc] finishTypeDescriptions[", i, "] mod=", m.module.name.s,
" t=", typeToString(t), " kind=", t.kind,
" itemId=", t.itemId.module, ".", t.itemId.item
if optSeqDestructors in m.config.globalOptions and t.skipTypes(abstractInst).kind == tySequence:
seqV2ContentType(m, t, check)
else:
@@ -1399,7 +1414,7 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
if m.config.cmd == cmdNifC:
m.icDataDefs.add (name, "")
m.icDataDefs.add (name, icNifName(m, origType))
proc genTypeInfoAux(m: BModule; typ, origType: PType, name: Rope;
info: TLineInfo) =
@@ -1687,8 +1702,13 @@ proc declareNimType(m: BModule; name: string; str: Rope, module: int) =
m.s[cfsTypeInit1].addArgument(hcrGlobal):
m.s[cfsTypeInit1].add("\"" & str & "\"")
else:
# cnif-mark the name: this extern declaration is the reference the
# def-retention check consults when the defining TU regenerates and
# the typeinfo cannot be re-demanded (type vanished) — the referencing
# TU must lose its reuse then instead of producing a link error
let declName = if m.config.cmd == cmdNifC: markCName(str) else: str
m.s[cfsStrData].addDeclWithVisibility(Extern):
m.s[cfsStrData].addVar(kind = Local, name = str, typ = nr)
m.s[cfsStrData].addVar(kind = Local, name = declName, typ = nr)
proc genTypeInfo2Name(m: BModule; t: PType): Rope =
var it = t
@@ -1828,7 +1848,7 @@ proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLin
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
if m.config.cmd == cmdNifC:
m.icDataDefs.add (name, "")
m.icDataDefs.add (name, icNifName(m, origType))
var flags = 0
if not canFormAcycle(m.g.graph, t): flags = flags or 1
@@ -1894,7 +1914,7 @@ proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineIn
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
if m.config.cmd == cmdNifC:
m.icDataDefs.add (name, "")
m.icDataDefs.add (name, icNifName(m, origType))
var flags = 0
if not canFormAcycle(m.g.graph, t): flags = flags or 1
@@ -2067,6 +2087,10 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
let marker = m.g.typeInfoMarker.getOrDefault(sig)
if marker.str != "":
when defined(icDbgRefc):
if "catchableerror" in marker.str:
echo "[icNti] ", marker.str, " in mod=", m.module.name.s,
" -> extern:globalMarker owner=", marker.owner
cgsym(m, "TNimType")
cgsym(m, "TNimNode")
declareNimType(m, "TNimType", marker.str, marker.owner)
@@ -2077,7 +2101,15 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
result = "NTI$1$2_" % [rope(typeToC(t)), rope($sig)]
m.typeInfoMarker[sig] = result
when defined(icDbgRefc):
template dbgNti(branch: string) =
if "catchableerror" in result:
echo "[icNti] ", result, " in mod=", m.module.name.s, " -> ", branch
else:
template dbgNti(branch: string) = discard
if m.config.cmd == cmdNifC and result in m.g.graph.icCachedDataDefs:
dbgNti "extern:cachedDataDefs"
# already defined inside a reused TU from the previous run
cgsym(m, "TNimType")
cgsym(m, "TNimNode")
@@ -2088,6 +2120,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
let old = m.g.graph.emittedTypeInfo.getOrDefault($result)
if old != FileIndex(0):
dbgNti "extern:emittedTypeInfo"
cgsym(m, "TNimType")
cgsym(m, "TNimNode")
declareNimType(m, "TNimType", result, old.int)
@@ -2095,6 +2128,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
var owner = t.skipTypes(typedescPtrs).itemId.module
if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
dbgNti "extern:ownerRouted"
# make sure the type info is created in the owner module
discard genTypeInfoV1(m.g.mods[owner], origType, info)
# reference the type info as extern here
@@ -2105,6 +2139,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
else:
owner = m.module.position.int32
dbgNti "DEFINED-HERE"
m.g.typeInfoMarker[sig] = (str: result, owner: owner)
#rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result)

View File

@@ -19,7 +19,7 @@ import
mangleutils, cbuilderbase, modulegraphs
from expanddefaults import caseObjDefaultBranch
from ast2nif import globalName, toNifFilename
from ast2nif import globalName, toNifFilename, icNifTypeName
from typekeys import modname
from std/algorithm import sort
import cnif
@@ -104,6 +104,15 @@ proc icNifName(m: BModule; s: PSym): string =
else:
result = ""
proc icNifName(m: BModule; t: PType): string =
## The type flavor: recorded next to RTTI data definitions so the
## def-retention check can re-demand the typeinfo of a regenerating TU's
## previous artifact (`genTypeInfo` is type-driven, not symbol-driven).
if m.config.cmd == cmdNifC:
result = icNifTypeName(t, m.config)
else:
result = ""
proc redirectToLiveModule(m: BModule, q: BModule): BModule =
## A module whose cached translation unit is reused never generates code,
## so a definition that `findPendingModule` routes into it must be emitted

View File

@@ -73,12 +73,14 @@ proc stripCnifMarks*(s: string): string =
inc i
const
CnifVersion* = "3"
CnifVersion* = "4"
## Artifact format version, stored in the meta head. Artifacts written
## by an older compiler lack the NIF names and the cref group the
## def-retention check needs (v2) or the cdeps group the fine-grained
## reuse gate needs (v3); `readCnifHeads` reports them as invalid so
## their TUs simply regenerate once.
## def-retention check needs (v2), the cdeps group the fine-grained
## reuse gate needs (v3), or the type NIF names and cnif-marked extern
## RTTI references the typeinfo flavor of the def-retention check
## needs (v4); `readCnifHeads` reports them as invalid so their TUs
## simply regenerate once.
proc cnifDefDirective*(name, flags, nifName: string): string =
CnifDefStart & name & CnifDefSep & flags & CnifDefSep & nifName & CnifDefEnd

View File

@@ -653,6 +653,18 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
conf: ConfigRef) =
var key = ""
var val = ""
# Record config-file switches so the `nim ic` driver can serialise them into a
# precompiled-config artifact and have its per-module child processes replay
# them instead of re-parsing the `nim.cfg` chain (and re-running `config.nims`
# in the VM) on every invocation. Only `passPP` (config-file) switches are
# captured; command-line switches are forwarded by the build graph as usual.
# Path-search switches are skipped: their net effect already lives in the
# resolved `searchPaths` the driver forwards as `--path`, and replaying their
# raw (often relative-to-config-dir) arguments here would misresolve.
if pass == passPP and switch.normalize notin
["path", "p", "nimblepath", "lazypath", "excludepath",
"nonimblepath", "clearnimblepath", "nimcache"]:
conf.icConfigSwitches.add (switch, arg)
case switch.normalize
of "eval":
expectArg(conf, switch, arg, pass, info)
@@ -937,6 +949,17 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icGroup.incl(canonicalizePath(conf, AbsoluteFile arg).string)
of "icproject":
# `nim m`/`nim nifc` only: the ORIGINAL project file (see options.icProject)
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icProject = canonicalizePath(conf, AbsoluteFile arg).string
of "icpreparsedconfig":
# `nim m`/`nim nifc` only: path of the precompiled-config artifact (see
# options.icPreparsedConfig). Read in `passCmd1`, before `loadConfigs`, so
# config loading can replay it instead of re-parsing the `nim.cfg` chain.
expectArg(conf, switch, arg, pass, info)
conf.icPreparsedConfig = arg
of "import":
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:

View File

@@ -10,8 +10,8 @@
## Generate a .build.nif file for nifmake from a Nim project.
## This enables incremental and parallel compilation using the `m` switch.
import std / [os, tables, sets, times, osproc, algorithm, strtabs]
import options, msgs, lineinfos, pathutils, condsyms
import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio]
import options, msgs, lineinfos, pathutils, condsyms, icconfig
import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder]
import "../dist/nimony/src/gear2" / modnames
@@ -415,13 +415,22 @@ proc parseImportPath(s: var Stream; t: var PackedToken): seq[string] =
let tag = pool.tags[t.tagId]
if tag == "infix":
t = next(s) # skip 'infix' tag
if t.kind == Ident: t = next(s) # skip the operator (`/`)
var op = ""
if t.kind == Ident:
op = pool.strings[t.litId]
t = next(s)
let left = parseImportPath(s, t)
let right = parseImportPath(s, t)
let prefix = if left.len == 1: left[0] else: ""
for r in right:
if prefix.len > 0: result.add prefix & "/" & r
else: result.add r
if op == "as":
# `import ../rlp/results as rlp_results`: the alias is not a path
# component — treating `as` like `/` produced the garbage path
# `../rlp/results/rlp_results`, silently dropping the dependency
result = left
else:
let prefix = if left.len == 1: left[0] else: ""
for r in right:
if prefix.len > 0: result.add prefix & "/" & r
else: result.add r
if t.kind == ParRi: t = next(s) # skip closing ')'
elif tag == "prefix":
# Relative import paths: `import ../dist/checksums/...` parses as
@@ -634,6 +643,21 @@ proc generateBuildFile(c: DepContext): string =
# buckets (and rejects calls as ambiguous that multi-dispatch accepts)
if optMultiMethods in c.config.globalOptions:
forwardedArgs.add "--multimethods:on"
# the children compile each MODULE as their own project file, which makes
# that module's package the "main package" and unfilters foreign-package
# diagnostics — a vendored package's hintAsError/warningAsError promotions
# then abort builds the whole-program compilation accepts. Forward the
# real project so children filter diagnostics identically.
forwardedArgs.add "--icproject:" & c.config.projectFull.string
# Precompiled config: serialise the driver's config once and have every
# child replay it instead of re-parsing the `nim.cfg` chain and re-running
# `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig`
# restores the old per-child config parsing (for bisecting a suspected
# config-replay divergence without clearing caches).
if not isDefined(c.config, "icNoPreparsedConfig"):
let cfgArtifact = nimcache / "ic_config.cfg.nif"
writeIcConfig(c.config, cfgArtifact)
forwardedArgs.add "--icPreparsedConfig:" & cfgArtifact
# Define nifler command
b.addTree "cmd"
@@ -857,8 +881,16 @@ proc commandIc*(conf: ConfigRef) =
rawMessage(conf, errGenerated, "project file not found: " & projectFile)
return
# Create nimcache directory
createDir(getNimcacheDir(conf).string)
# Create nimcache directory; start from a clean one when its format
# stamp is absent or outdated (see `icFormatVersion`)
let cacheDir = getNimcacheDir(conf).string
createDir(cacheDir)
let versionFile = cacheDir & "/ic.version"
let stamp = if fileExists(versionFile): readFile(versionFile) else: ""
if stamp != icFormatVersion:
removeDir(cacheDir)
createDir(cacheDir)
writeFile(versionFile, icFormatVersion)
var c = DepContext(
config: conf,
@@ -901,19 +933,56 @@ proc commandIc*(conf: ConfigRef) =
# Process dependencies
traverseDeps(c, rootPair, rootNode)
# Generate build file
let buildFile = generateBuildFile(c)
rawMessage(conf, hintSuccess, "generated: " & buildFile)
# Discovery loop: imports GENERATED by macros (chronicles builds
# `import chronicles/textlines` via parseStmt from the chronicles_sinks
# define) are invisible to the static scanner. A failing `nim m` child
# records "missing-path \t importer-path" in icmissing.txt; we add the
# module (and an edge from its importer) to the graph and rerun —
# nifmake's mtime pruning keeps completed work.
let missingFile = getNimcacheDir(conf).string & "/icmissing.txt"
removeFile missingFile
var rounds = 0
while true:
# Generate build file
let buildFile = generateBuildFile(c)
rawMessage(conf, hintSuccess, "generated: " & buildFile)
# Automatically run nifmake
let nifmake = findNifmake()
if nifmake.len == 0:
rawMessage(conf, hintSuccess, "run: nifmake run " & buildFile)
else:
# Automatically run nifmake
let nifmake = findNifmake()
if nifmake.len == 0:
rawMessage(conf, hintSuccess, "run: nifmake run " & buildFile)
break
let cmd = quoteShell(nifmake) & " run " & quoteShell(buildFile)
rawMessage(conf, hintExecuting, cmd)
let exitCode = execShellCmd(cmd)
if exitCode != 0:
if exitCode == 0: break
var discovered = false
inc rounds
if rounds <= 20 and fileExists(missingFile):
for line in lines(missingFile):
let parts = line.split('\t')
if parts.len != 2 or parts[0].len == 0: continue
let pair = c.toPair(parts[0])
let importerIdx = c.processedModules.getOrDefault(
c.toPair(parts[1]).modname, -1)
var idx = c.processedModules.getOrDefault(pair.modname, -1)
if idx == -1:
let newNode = Node(files: @[pair], id: c.nodes.len)
if c.systemNodeId >= 0:
newNode.deps.add c.systemNodeId
c.processedModules[pair.modname] = newNode.id
c.nodes.add newNode
idx = newNode.id
traverseDeps(c, pair, newNode)
discovered = true
if importerIdx >= 0 and idx >= 0 and idx notin c.nodes[importerIdx].deps:
# the build-graph edge the scanner could not see: forces the
# discovered module to be built before its importer re-sems
c.nodes[importerIdx].deps.add idx
discovered = true
removeFile missingFile
if not discovered:
rawMessage(conf, errGenerated, "nifmake failed with exit code: " & $exitCode)
break
else:
rawMessage(conf, errGenerated, "nim ic not available in bootstrap build")

View File

@@ -307,6 +307,7 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
elif not (owner.typ.isClosure or owner.isNimcall and not owner.isExplicitCallConv or isEnv):
localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
[s.name.s, owner.name.s, $owner.typ.callConv])
unsealForTransform(owner.typ)
incl(owner.typ, tfCapturesEnv)
if not isEnv:
owner.typ.callConv = ccClosure

View File

@@ -378,6 +378,9 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
conflictsWith: TLineInfo, note = errGenerated) =
## Emit a redefinition error if in non-interactive mode
if c.config.cmd != cmdInteractive:
when defined(icDbgRefc):
echo "[icRedef] ", s
echo getStackTrace()
localError(c.config, info, note,
"redefinition of '$1'; previous declaration here: $2" %
[s, c.config $ conflictsWith])

View File

@@ -105,18 +105,39 @@ type
# they were in its previous artifact and a
# reused TU still references them (the
# backend def-migration check)
icPreserveTypeInfos*: Table[int, seq[PType]] # the same for RTTI data
# definitions, which are type-driven: the
# type whose `genTypeInfo` the TU must
# re-demand
icImplDeps*: IntSet # NeedsImpl edge tracking under `nim m`:
# module ids (FileIndex) whose routine BODIES
# this compilation consumed at compile time
# (VM-compiled or getImpl'ed). Written to the
# `.edges` sidecar; deps.nim then gates the
# dependent on those modules' IMPL cookie
# instead of the iface cookie, so e.g.
# `const x = dep.foo()` re-sems when foo's
# body changes. Bodies with inline semantics
# (templates/macros/generics/iterators) need
# no tracking: they are part of the iface
# cookie itself.
# this compilation consumed at compile time.
# Written to the `.edges` sidecar; deps.nim
# then gates the dependent on those modules'
# IMPL cookie instead of the iface cookie, so
# e.g. `const x = dep.foo()` re-sems when foo's
# body changes. Uniform across body-access
# kinds — the iface cookie hashes signatures
# ONLY (see ast2nif.cookieSd), so every body
# consumer records an edge here: VM-compiled /
# getImpl'ed bodies (recordIcImplDep from vm/
# vmgen), expanded templates (semTemplateExpr)
# and instantiated generics (generateInstance).
# Inline iterators / `inline` procs are NOT
# tracked: they are inlined at codegen, where
# the nifc backend's NIF-mtime invalidation
# already re-codegens their users.
icQualIfaces*: IntSet # module positions whose interface tables were
# populated ONLY for qualified access through a
# module re-export (`import x; export x`); the
# Iface.module stays nil so a later direct
# import still takes the full load path
inVMTransform*: int # >0 while the VM compiles a routine body
# (vmgen.genProc's transformBody): hooks lifted
# there (e.g. for closure-env types of LOADED
# routines) are process-local VM artifacts —
# serializing them would embed references to
# derived env-field syms that no module defines
packageSyms*: TStrTable
deps*: IntSet # the dependency graph or potentially its transitive closure.
@@ -285,6 +306,18 @@ iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
if s != nil:
yield s
proc reexportedModuleSyms*(g: ModuleGraph; m: PSym): seq[(string, string)] =
## (name, NIF module suffix) of MODULE syms in `m`'s interface — these are
## re-exports (`import x; export x`, added by `reexportSym`) acting as
## qualifiers (`m.x.sym`). Consumed by the NIF writer; semExport does not
## put them into the nkExportStmt children, so the AST walk cannot see them.
result = @[]
var seen = initIntSet()
for s in g.ifaces[m.position].interf.data:
if s != nil and s.kind == skModule and s.position != m.position and
not seen.containsOrIncl(s.position):
result.add (s.name.s, cachedModuleSuffix(g.config, FileIndex s.position))
proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
let importHidden = optImportHidden in m.options
result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name)
@@ -346,6 +379,15 @@ proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp;
# Key-based deduplication for opsLog: different type objects (e.g. canon vs
# orig) can have different itemIds but the same structural key.
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
if g.inVMTransform > 0 and g.config.cmd == cmdM:
# hook lifted while the VM compiles a routine body (closure-env types of
# loaded routines): register it for in-process lookup but keep it out of
# the serialized log — it is a process-local artifact whose type graph
# references derived env-field syms that no module's NIF defines
if g.loadedOps[op].getOrDefault(key) == nil:
g.loadedOps[op][key] = value
g.attachedOps[op][t.itemId] = value
return
let existing = g.loadedOps[op].getOrDefault(key)
if existing == nil:
# Stamp the entry with the module whose compilation produced the hook
@@ -941,6 +983,39 @@ when not defined(nimKochBootstrap):
registerLoadedHooks(g, precomp.logOps)
for d in precomp.deps: stack.add d
proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym =
## A re-exported MODULE (`import x; export x`) acts as a qualifier in the
## re-exporting module's interface (`asmm.x86.nd`). Reconstruct a module
## symbol for it and make its interface tables available for qualified
## lookup (`someSym` reads `g.ifaces[position]`) — WITHOUT registering
## the module: `Iface.module` stays nil so a later direct import still
## takes the full load path (replayStateChanges etc.).
var isKnown = false
let fIdx = g.config.registerNifSuffix(msuffix, isKnown)
if fIdx.int >= g.ifaces.len: setLen(g.ifaces, fIdx.int + 1)
if g.ifaces[fIdx.int].module != nil and
g.ifaces[fIdx.int].module.name.s == mname:
# properly registered already (directly imported earlier): reuse it
return g.ifaces[fIdx.int].module
result = PSym(kindImpl: skModule, itemId: itemId(int32(fIdx), 0'i32),
name: getIdent(g.cache, mname),
infoImpl: newLineInfo(fIdx, 1, 1),
positionImpl: int(fIdx))
setOwner(result, getPackage(g.config, g.cache, fIdx))
if g.ifaces[fIdx.int].module == nil and
not g.icQualIfaces.containsOrIncl(fIdx.int):
var interf = initStrTable()
var interfHidden = initStrTable()
let precomp = loadNifModule(ast.program, ModuleSuffix(msuffix),
interf, interfHidden, {})
# chains: the re-exported module may itself re-export modules
for (n2, s2) in precomp.reexportedModules:
let inner = materializeReexportedModule(g, n2, s2)
if inner != nil:
strTableAdd(interf, inner)
g.ifaces[fIdx.int].interf = interf
g.ifaces[fIdx.int].interfHidden = interfHidden
proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex;
flags: set[LoadFlag] = {}): PrecompiledModule =
## Returns 'nil' if the module needs to be recompiled.
@@ -966,6 +1041,10 @@ when not defined(nimKochBootstrap):
g.ifaces[fileIdx.int].interf,
g.ifaces[fileIdx.int].interfHidden, flags)
result.module = m
for (mname, msuffix) in result.reexportedModules:
let ms = materializeReexportedModule(g, mname, msuffix)
if ms != nil:
strTableAdd(g.ifaces[fileIdx.int].interf, ms)
# Mark module as cached
g.cachedMods.incl fileIdx.int

View File

@@ -149,11 +149,30 @@ proc enforceDefRetention(g: ModuleGraph; mainPos: int;
# targets the un-reuse fallback. Demand-side dedup (`declaredThings`,
# the cached/claim shortcuts) makes redundant re-demands cheap.
clear g.icPreserveDefs
clear g.icPreserveTypeInfos
var unreuse = initHashSet[int]()
for src in sources.items:
template check(defseq) =
for d in defseq:
if d.cname notin cachedDefs:
if d.nifname.startsWith("`t"):
# an RTTI definition: re-demand is type-driven (`genTypeInfo`),
# there is no symbol to resolve
let typ = resolveGlobalType(ast.program, d.nifname)
if typ != nil:
g.icPreserveTypeInfos.mgetOrPut(src.target, @[]).add typ
if icDebug:
stderr.writeLine "[icRetain] preserve typeinfo " & d.cname
continue
elif d.cname in refdBy:
# type vanished: un-reuse the TUs that still reference it
for tu in refdBy[d.cname]: unreuse.incl tu
if icDebug:
stderr.writeLine "[icRetain] cannot re-demand typeinfo " &
d.cname & "; un-reusing referencing TUs"
continue
else:
continue
var sym: PSym = nil
if d.nifname.len > 0:
sym = resolveGlobalSym(ast.program, d.nifname)
@@ -406,6 +425,16 @@ proc eagerHookCandidate(sym: PSym): bool =
let pt = typ.n[i].typ
if pt == nil: return false
if iterOverType(pt, isMetaIter, nil): return false
# a `=dup` of an imported type returns it by value; for "lying" importc
# typedefs like `jmp_buf` (declared as `object`, really a C array) that
# signature does not compile. Demand-driven codegen never demands such
# sem-bookkeeping hooks (under refc nothing dups a `C_JmpBuf`), and no
# working artifact can call one — its prototype would be the same
# invalid C — so they are safe to skip.
let ret = typ.returnType
if ret != nil:
let r = ret.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct})
if r.sym != nil and sfImportc in r.sym.flags: return false
true
proc finishModule(g: ModuleGraph; bmod: BModule) =
@@ -472,6 +501,9 @@ proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
if g.icPreserveDefs.hasKey(moduleId):
for sym in g.icPreserveDefs[moduleId]:
requestAnyDef(bmod, sym)
if g.icPreserveTypeInfos.hasKey(moduleId):
for t in g.icPreserveTypeInfos[moduleId]:
discard genTypeInfo(g.config, bmod, t, unknownLineInfo)
proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
## Main entry point for NIF-based C code generation.

View File

@@ -11,7 +11,7 @@
import
llstream, commands, msgs, lexer, ast,
options, idents, wordrecg, lineinfos, pathutils, scriptconfig
options, idents, wordrecg, lineinfos, pathutils, scriptconfig, icconfig
import std/[os, strutils, strtabs]
@@ -246,6 +246,12 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile
proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) =
setDefaultLibpath(conf)
# `nim ic` children replay the precompiled config the driver recorded once,
# instead of re-reading the `nim.cfg` chain and re-running `config.nims` in the
# VM. A missing/format-incompatible artifact returns false: fall through to
# normal config loading so an older child or a deleted cache still works.
if conf.icPreparsedConfig.len > 0 and applyIcConfig(conf, conf.icPreparsedConfig):
return
template readConfigFile(path) =
let configPath = path
conf.currentConfigDir = configPath.splitFile.dir.string

View File

@@ -29,6 +29,18 @@ const
nimEnableCovariance* = defined(nimEnableCovariance)
icFormatVersion* = "2"
## Version of the IC cache format (the sem-NIF module layout written by
## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
## that layout changes: `commandIc` wipes a nimcache whose `ic.version`
## stamp differs, instead of letting a newer reader mis-parse records
## written by an older compiler (nifmake's rebuild check is mtime-only
## and knows nothing about format changes).
## v2: iface cookie hashes routine SIGNATURES only (no inline-semantics
## body folding); body access now records a NeedsImpl edge instead. A v1
## cache mixes body-sensitive and body-insensitive cookies, so it must be
## wiped rather than warm-rebuilt.
type # please make sure we have under 32 options
# (improves code efficiency a lot!)
TOption* = enum # **keep binary compatible**
@@ -386,6 +398,24 @@ type
# recursion resolves in-memory) and each gets its NIF
# written, instead of being loaded from a precompiled
# NIF. See `compiler/deps.nim` (SCC grouping).
icProject*: string # under `nim m`/`nim nifc`: absolute path of the
# ORIGINAL project file. The child's own project file
# is the module being compiled, which would make that
# module's package the "main package" and unfilter
# foreign-package diagnostics; the real project
# restores whole-program filtering semantics.
icPreparsedConfig*: string # under `nim m`/`nim nifc`: path of the precompiled
# config artifact written once by the `nim ic` driver.
# When set, `loadConfigs` replays the recorded
# config-file switches from it instead of re-reading
# the `nim.cfg` chain and re-running `config.nims`
# (which the VM makes expensive) per subprocess.
icConfigSwitches*: seq[tuple[switch, arg: string]]
# the config-file (`passPP`) switches applied while
# loading config, in order. Recorded by every nim
# process; only the `ic` driver serialises them.
# Path-search switches are excluded — the driver
# forwards the resolved `searchPaths` as `--path`.
spellSuggestMax*: int # max number of spelling suggestions for typos
cppDefines*: HashSet[string] # (*)

View File

@@ -270,7 +270,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
var implDeps: seq[int] = @[]
for id in graph.icImplDeps: implDeps.add id
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps)
replayActions, implDeps, reexportedModuleSyms(graph, module))
result = true
@@ -304,6 +304,18 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
let precomp = moduleFromNifFile(graph, fileIdx)
if precomp.module == nil:
let nifPath = toNifFilename(graph.config, fileIdx)
# Record the miss for `nim ic`'s discovery loop: imports GENERATED
# by macros (e.g. chronicles' parseStmt("import chronicles/textlines"),
# driven by the chronicles_sinks define) are invisible to the static
# dependency scanner. The parent reads this file, adds the module —
# plus an edge from this importer — to the build graph and reruns.
try:
let f = open(getNimcacheDir(graph.config).string & "/icmissing.txt", fmAppend)
f.writeLine(toFullPath(graph.config, fileIdx) & "\t" &
graph.config.projectFull.string)
f.close()
except IOError, OSError:
discard
globalError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) &
" (expected: " & nifPath & ")")
@@ -382,7 +394,14 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
conf.projectMainIdx2 = projectFile
let packSym = getPackage(graph, projectFile)
var packSym = getPackage(graph, projectFile)
if graph.config.cmd in {cmdM, cmdNifC} and graph.config.icProject.len > 0:
# per-module IC children: the process' project file is the MODULE being
# compiled, which would make its package the "main package" and unfilter
# foreign-package diagnostics (a vendored package's hintAsError promotion
# then aborts builds the whole-program compilation accepts). Use the
# original project, forwarded by deps.nim via --icproject.
packSym = getPackage(graph, fileInfoIdx(graph.config, AbsoluteFile graph.config.icProject))
graph.config.mainPackageId = packSym.getPackageId
graph.importStack.add projectFile

View File

@@ -373,6 +373,13 @@ proc addConverter*(c: PContext, conv: PSym) =
proc addConverterDef*(c: PContext, conv: PSym) =
addConverter(c, conv)
# record the definition for IC: the loader rebuilds Iface.converters from
# the NIF's (repconverter ...) entries (moduleFromNifFile); without the log
# entry a loaded module's converters were invisible to importers and
# implicit conversions silently stopped matching (e.g. faststreams'
# InputStreamHandle -> InputStream at toml_serialization call sites)
c.graph.opsLog.add LogEntry(kind: ConverterEntry, module: c.module.position,
key: "", sym: conv)
proc addPureEnum*(c: PContext, e: PSym) =
assert e != nil

View File

@@ -27,6 +27,11 @@ const
proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
flags: TExprFlags = {}; expectedType: PType = nil): PNode =
rememberExpansion(c, n.info, s)
# IC: this expands `s`'s body into the current module's sem, so the module
# depends on that body — record a NeedsImpl (strong) edge to `s`'s module.
# The iface cookie hashes only signatures now, so a template body edit moves
# only the impl cookie, and just the modules that expanded it re-sem.
recordIcImplDep(c.graph, s)
let info = getCallLineInfo(n)
markUsed(c, info, s)
onUse(info, s)
@@ -57,6 +62,16 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
elif {efWantStmt, efAllowStmt} * flags != {}:
result.typ = newTypeS(tyVoid, c)
else:
when defined(icDbgRefc):
echo "[icNoType] semOperand: ", renderTree(result, {renderNoComments}),
" kind=", result.kind,
(if result.kind in {nkCall, nkCommand} and result[0].kind == nkSym:
" calleeTyp=" & (if result[0].sym.typ == nil: "NIL" else:
$result[0].sym.typ.kind & " ret=" &
(if result[0].sym.typ.returnType == nil: "NIL"
else: $result[0].sym.typ.returnType.kind))
else: "")
echo getStackTrace()
localError(c.config, n.info, errExprXHasNoType %
renderTree(result, {renderNoComments}))
result.typ = errorType(c)
@@ -83,6 +98,17 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType
if result.typ == nil and efInTypeof in flags:
result.typ = c.voidType
elif result.typ == nil or result.typ == c.enforceVoidContext:
when defined(icDbgRefc):
echo "[icNoType] semExprWithType: ", renderTree(result, {renderNoComments}),
" kind=", result.kind,
(if result.kind in {nkCall, nkCommand} and result[0].kind == nkSym:
" callee=" & result[0].sym.name.s &
" calleeTyp=" & (if result[0].sym.typ == nil: "NIL" else:
$result[0].sym.typ.kind & " ret=" &
(if result[0].sym.typ.returnType == nil: "NIL"
else: $result[0].sym.typ.returnType.kind))
else: "")
echo getStackTrace()
localError(c.config, n.info, errExprXHasNoType %
renderTree(result, {renderNoComments}))
result.typ = errorType(c)
@@ -2144,6 +2170,12 @@ proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode =
if c.p.owner.kind notin {skMacro, skTemplate} and
c.p.resultSym != nil and c.p.resultSym.typ.isMetaType:
when defined(icDbgRefc):
echo "[icMetaRet] meta result type for ", c.p.owner.name.s, ": ",
typeToString(c.p.resultSym.typ), " kind=", c.p.resultSym.typ.kind,
" flags=", c.p.resultSym.typ.flags,
" uid=", c.p.resultSym.typ.uniqueId.module, ".", c.p.resultSym.typ.uniqueId.item,
" state=", c.p.resultSym.typ.state
if isEmptyType(result.typ):
# we inferred a 'void' return type:
c.p.resultSym.typ = errorType(c)

View File

@@ -119,11 +119,44 @@ proc freshGenSyms(c: PContext; n: PNode, owner, orig: PSym, symMap: var SymMappi
proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind)
proc aliasLoadedTypedescParams(c: PContext, instantiated, orig: PSym): bool =
## When the generic being instantiated had its body LOADED from a NIF (only
## `nim m`/`nim nifc`, only for a generic owned by another module), that body
## re-sems from plain identifiers — ast2nif serialises locals/params as idents,
## not `nkSym`. A `T: typedesc[...]` param referenced as a type must then
## resolve `T` to the bound type, but the instantiated skParam carries the
## concrete type `instantiateProcType` typedesc-skipped it to, which an ident
## lookup cannot use as a type name. Shadow each such param with an `skType`
## alias of the same name in a fresh scope layer (the alias is exactly how Nim
## models "this name denotes a type"). In-process bodies reach the param as
## `nkSym` and never take this path, hence the command gate.
##
## Returns true iff a scope layer was opened; the caller must `closeScope`.
if c.config.cmd notin {cmdM, cmdNifC} or orig == nil or
orig.itemId.module == c.module.position or
orig.typ == nil or orig.typ.n == nil:
return false
result = false
let procParams = instantiated.typ.n
for i in 1..<min(procParams.len, orig.typ.n.len):
if orig.typ.n[i].kind != nkSym: continue
let origParamTyp = orig.typ.n[i].sym.typ
if origParamTyp != nil and origParamTyp.kind == tyTypeDesc and
tfUnresolved in origParamTyp.flags:
if not result:
openScope(c)
result = true
let p = procParams[i].sym
let alias = newSym(skType, p.name, c.idgen, instantiated, p.info)
alias.typ = p.typ
addDecl(c, alias)
proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
if n[bodyPos].kind != nkEmpty:
let procParams = result.typ.n
for i in 1..<procParams.len:
addDecl(c, procParams[i].sym)
let aliasLayer = aliasLoadedTypedescParams(c, result, orig)
maybeAddResult(c, result, result.ast)
inc c.inGenericInst
@@ -152,6 +185,7 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
excl(result, sfForward)
trackProc(c, result, result.ast[bodyPos])
dec c.inGenericInst
if aliasLayer: closeScope(c)
proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
for i in 0..<c.generics.len:
@@ -281,6 +315,12 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
let param = copySym(oldParam, c.idgen)
setOwner(param, prc)
param.typ = paramType
when defined(icDbgRefc):
echo "[icInst] ", prc.name.s, " param ", oldParam.name.s,
": ", typeToString(resulti), " (kind=", resulti.kind,
" uid=", resulti.uniqueId.module, ".", resulti.uniqueId.item,
" flags=", resulti.flags, ") -> ", typeToString(paramType),
" (kind=", paramType.kind, ")"
# The default value is instantiated and fitted against the final
# concrete param type. We avoid calling `replaceTypeVarsN` on the
@@ -382,6 +422,11 @@ proc generateInstance(c: PContext, fn: PSym, pt: LayeredIdTable,
## parameters to their concrete types within the generic instance.
# no need to instantiate generic templates/macros:
internalAssert c.config, fn.kind notin {skMacro, skTemplate}
# IC: instantiating `fn` consumes its generic body in the current module's
# sem — record a NeedsImpl (strong) edge to `fn`'s module. The iface cookie
# hashes only signatures now, so a generic body edit moves only the impl
# cookie, and just the modules that instantiated it re-sem.
recordIcImplDep(c.graph, fn)
# generates an instantiated proc
if c.instCounter > 50:
globalError(c.config, info, "generic instantiation too nested")

View File

@@ -1361,7 +1361,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
for i in 0..<paramType.len - 1:
if paramType[i].kind == tyStatic:
var staticCopy = paramType[i].exactReplica
var staticCopy = paramType[i].exactReplica(c.idgen)
staticCopy.incl tfInferrableStatic
result.rawAddSon staticCopy
else:
@@ -2148,7 +2148,7 @@ proc semTypeIdent(c: PContext, n: PNode): PSym =
localError(c.config, n.info, errTypeExpected)
return errorSym(c, n)
result = result.typ.sym.copySym(c.idgen)
result.typ = exactReplica(result.typ)
result.typ = exactReplica(result.typ, c.idgen)
result.typ.incl tfUnresolved
if result.kind == skGenericParam:

View File

@@ -272,10 +272,17 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0; expectedType: PT
if n == nil: return
result = copyNode(n)
if n.typ != nil:
if n.typ.kind == tyFromExpr:
var nodeTyp = n.typ
if nodeTyp.kind == tyFromExpr:
# type of node should not be evaluated as a static value
n.typ.incl tfNonConstExpr
result.typ = replaceTypeVarsT(cl, n.typ)
if nodeTyp.state == Sealed:
# IC: do not brand the loaded shared original — a tyFromExpr is a
# placeholder that `replaceTypeVarsT` resolves away, so the copy
# carries no identity later comparisons could miss (mirrors
# `instantiateProcType`)
nodeTyp = copyType(nodeTyp, cl.c.idgen, nodeTyp.owner)
nodeTyp.incl tfNonConstExpr
result.typ = replaceTypeVarsT(cl, nodeTyp)
checkMetaInvariants(cl, result.typ)
case n.kind
of nkNone..pred(nkSym), succ(nkSym)..nkNilLit:
@@ -387,6 +394,13 @@ proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
# don't bind `auto` return type to a previous binding of `auto`
return nil
result = cl.typeMap.lookup(t)
when defined(icDbgRefc):
if t.kind in {tyGenericParam, tyTypeDesc}:
echo "[icBind] lookup ", t.kind, " ", typeToString(t), " uid=", t.uniqueId.module, ".",
t.uniqueId.item, " itemId=", t.itemId.module, ".", t.itemId.item,
" state=", t.state, " flags=", t.flags, " -> ",
(if result != nil: typeToString(result) else: "MISS"),
" allowMeta=", cl.allowMetaTypes
if result == nil:
if cl.allowMetaTypes or tfRetType in t.flags: return
localError(cl.c.config, t.sym.info, "cannot instantiate: '" & typeToString(t) & "'")
@@ -401,7 +415,7 @@ proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
# XXX: relying on allowMetaTypes is a kludge
if cl.allowMetaTypes:
result = t.exactReplica
result = t.exactReplica(cl.c.idgen)
else:
result = copyType(t, cl.c.idgen, t.owner)
copyTypeProps(cl.c.graph, cl.c.idgen.module, result, t)

View File

@@ -135,6 +135,11 @@ proc put(c: var TCandidate, key, val: PType) {.inline.} =
writeStackTrace()
if c.c.module.name.s == "temp3":
echo "binding ", key, " -> ", val
when defined(icDbgRefc):
if key.kind in {tyGenericParam, tyTypeDesc}:
echo "[icBind] put ", key.kind, " ", typeToString(key), " uid=", key.uniqueId.module, ".",
key.uniqueId.item, " itemId=", key.itemId.module, ".", key.itemId.item,
" state=", key.state, " -> ", typeToString(val)
put(c.bindings, key, val.skipIntLit(c.c.idgen))
proc typeRel*(c: var TCandidate, f, aOrig: PType,
@@ -911,7 +916,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
case typ.kind
of tyStatic:
param = paramSym skConst
param.typ = typ.exactReplica
param.typ = typ.exactReplica(m.c.idgen)
#copyType(typ, c.idgen, typ.owner)
if typ.n == nil:
param.typ.incl tfInferrableStatic
@@ -919,7 +924,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
param.ast = typ.n
of tyFromExpr:
param = paramSym skVar
param.typ = typ.exactReplica
param.typ = typ.exactReplica(m.c.idgen)
#copyType(typ, c.idgen, typ.owner)
else:
param = paramSym skType
@@ -972,7 +977,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
if ff.kind == tyUserTypeClassInst:
result = generateTypeInstance(c, m.bindings, typeClass.sym.info, ff)
else:
result = ff.exactReplica
result = ff.exactReplica(m.c.idgen)
#copyType(ff, c.idgen, ff.owner)
result.n = checkedBody
@@ -2666,7 +2671,7 @@ proc staticAwareTypeRel(m: var TCandidate, f: PType, arg: var PNode): TTypeRelat
# The ast of the type does not point to the symbol.
# Without this we will never resolve a `static proc` with overloads
let copiedNode = copyNode(arg)
copiedNode.typ = exactReplica(copiedNode.typ)
copiedNode.typ = exactReplica(copiedNode.typ, m.c.idgen)
copiedNode.typ.n = arg
arg = copiedNode
typeRel(m, f, arg.typ)

View File

@@ -153,7 +153,12 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
for a in t.sonsImpl:
c.typeKey a, flags, conf
of tyDistinct:
if CoDistinct in flags:
if t.sonsImpl.len == 0:
# a bare `distinct` typeclass (e.g. `foo(distinct, ...)` matched
# against a `T: type` param) has no base type to key — it IS its kind
withTree c.m, toNifTag(t.kind):
c.m.addEmpty()
elif CoDistinct in flags:
if t.symImpl != nil: symKey(c, t.symImpl, conf)
if t.symImpl == nil or tfFromGeneric in t.flagsImpl:
c.typeKey t.sonsImpl[^1], flags, conf
@@ -238,9 +243,14 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
if t.typeInstImpl != nil:
# prevent against infinite recursions here, see bug #8883:
let inst = t.typeInstImpl
if inst.state == Partial:
# a lazily-loaded typeInst stub has no sons until forced in
assert c.tl != nil
c.tl(inst)
t.typeInstImpl = nil # IC: spurious writes are ok since we set it back immediately
assert inst.kind == tyGenericInst
c.typeKey inst.sonsImpl[0], flags, conf
if inst.sonsImpl.len > 0:
c.typeKey inst.sonsImpl[0], flags, conf
for i in 1..<inst.sonsImpl.len-1:
# Match sighashes: generic-instantiation arguments are keyed with
# `CoDistinct` so distinct args are not collapsed to their base.
@@ -297,7 +307,11 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
for i in 0..<t.sonsImpl.len:
c.typeKey t.sonsImpl[i], flags+{CoIgnoreRange}, conf
of tyRange:
if CoIgnoreRange notin flags:
if t.sonsImpl.len == 0:
# bare `range` typeclass: no base type, key the kind alone
withTree c.m, toNifTag(t.kind):
c.m.addEmpty()
elif CoIgnoreRange notin flags:
withTree c.m, toNifTag(t.kind):
c.treeKey(t.nImpl, {}, conf)
c.typeKey(t.sonsImpl[^1], flags, conf)
@@ -342,8 +356,12 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
if tfVarargs in t.flagsImpl: c.m.addIdent "´varargs"
of tyArray:
withTree c.m, toNifTag(t.kind):
c.typeKey(t.sonsImpl[^1], flags-{CoIgnoreRange}, conf)
c.typeKey(t.sonsImpl[0], flags-{CoIgnoreRange}, conf)
if t.sonsImpl.len == 0:
# bare `array` typeclass: no element/index types
c.m.addEmpty()
else:
c.typeKey(t.sonsImpl[^1], flags-{CoIgnoreRange}, conf)
c.typeKey(t.sonsImpl[0], flags-{CoIgnoreRange}, conf)
else:
withTree c.m, toNifTag(t.kind):
for i in 0..<t.sonsImpl.len:

View File

@@ -2477,7 +2477,9 @@ proc genProc(c: PCtx; s: PSym): VmProcInfo =
c.procToCodePos[s.id] = result
# thanks to the jmp we can add top level statements easily and also nest
# procs easily:
inc c.graph.inVMTransform
let body = transformBody(c.graph, c.idgen, s, if isCompileTimeProc(s): {} else: {useCache})
dec c.graph.inVMTransform
let procStart = c.xjmp(body, opcJmp, 0)
var p = PProc(blocks: @[], sym: s)
let oldPrc = c.prc

View File

@@ -419,7 +419,11 @@ proc bootic(args: string, skipIntegrityCheck: bool) =
## The 3-step fixed-point check is kept: a successful run proves the compiler
## can compile itself under IC and reproduces a stable binary.
var output = "compiler" / "nim".exe
var finalDest = "bin" / "nim".exe
# Deliberately NOT `bin/nim`: `bootic` must not clobber the development
# compiler (that would replace a fast release `bin/nim` with bootic's build
# and slow every later `koch`/`nim` invocation). The IC-bootstrapped binary
# lands at `bin/nim_ic` instead; `bin/nim` is only ever read (via findStartNim).
var finalDest = "bin" / "nim_ic".exe
let smartNimcache = (if "release" in args or "danger" in args: "nimcache/ric_" else: "nimcache/dic_") &
hostOS & "_" & hostCPU
@@ -444,7 +448,7 @@ proc bootic(args: string, skipIntegrityCheck: bool) =
[nimi, smartNimcache, args]
if sameFileContent(output, i.thVersion):
copyExe(output, finalDest)
echo "executables are equal: SUCCESS!"
echo "executables are equal: SUCCESS! (IC-bootstrapped compiler: ", finalDest, ")"
return
copyExe(output, (i+1).thVersion)
copyExe(output, finalDest)