mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-08-29 01:51:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

IC refactor (#25927)

Browse Source
This commit is contained in:
Andreas Rumpf
2026-06-25 23:20:34 +02:00
committed by GitHub
parent 056eeeae30
commit 5688a122f1
44 changed files with 1585 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
compiler/ast.nim
View File

@@ -831,6 +831,10 @@ proc newSymNode*(sym: PSym): PNode =
result = newNode(nkSym)
result.sym = sym
result.typField = sym.typ
if result.typField == nil and nifcBackendActive:
# See the two-arg overload in astdef: in the NIF backend cg stage a sym node
# built from a not-yet-typed stub must track the symbol's type lazily.
result.flags.incl nfLazyType
result.info = sym.info
proc newOpenSym*(n: PNode): PNode {.inline.} =

441
compiler/ast2nif.nim
View File

@@ -26,6 +26,13 @@ import typekeys
import ic / [enum2nif]
const SysModuleSuffix* = "@sys"
const BackendLocalMarker* = "@bk"
## Suffix marker for a PROCESS-LOCAL backend-minted entity (a closure `:env`
## type/obj/field/hidden-param minted while the VM compiles a routine body to
## run a macro). Such entities have no stable cross-process identity, so each
## module that references one emits its OWN module-local def named
## `…<thisModuleSuffix>@bk` and the loader homes it to the reading module with
## a `backendItemId` (disjoint from real ids). See transf.transformBody.
## Reserved module-suffix sentinel for module-less magic singleton types — the
## `nil` type is created via `newSysType` with the graph idgen, whose `module`
## can be `-1` (e.g. during VM const-eval before a real module is current), so
@@ -204,6 +211,13 @@ type
# can keep mutating its still-live query targets
writtenPackages: HashSet[string]
depSuffixes: HashSet[string] # module suffixes already emitted as `(import ...)` deps
emittedBackendTypes: HashSet[int32] # backend-local type items already def'd this module
emittedBackendSyms: HashSet[int32] # backend-local sym items already def'd this module
lowering: bool # serializing the `lower` stage's whole-module `.t.nif`
emittedFieldSyms: HashSet[ItemId] # lowering: derived env-field syms already def'd
inTypeReclist: int # >0 while writing a type's OWN reclist: fields must be SELF-CONTAINED
# defs (the type can be seek-loaded in isolation), not entry-deduped uses
proc isLocalSym(sym: PSym): bool {.inline.} =
## Every symbol is emitted as a *global* (module-suffixed) name so that its
@@ -232,7 +246,17 @@ const
proc toNifSymName(w: var Writer; sym: PSym): string =
## Generate NIF name for a symbol: local names are `ident.disamb`,
## global names are `ident.disamb.moduleSuffix`
assert not sym.itemId.isBackendMinted
if sym.itemId.isBackendMinted:
# Process-local backend sym (closure env field / hidden `:env` param minted
# during a VM transform): re-home to the current module with the `@bk`
# marker so each referencing module self-contains it. See transformBody.
result = sym.name.s
result.add '.'
result.addInt sym.disamb
result.add '.'
result.add modname(w.currentModule, w.infos.config)
result.add BackendLocalMarker
return
result = sym.name.s
if sym.kindImpl == skPackage:
result.add PkgMarker
@@ -254,6 +278,11 @@ proc globalName*(sym: PSym; config: ConfigRef): string =
result.addInt sym.disamb
result.add '.'
result.add modname(sym.itemId.module, config)
# A loaded process-local backend sym keeps its `@bk` marker in the NIF name
# (the index/`c.syms` tables are keyed by it); mirror toNifSymName so name-based
# lookups via globalName don't miss (KeyError `:env.N.<mod>` without the marker).
if sym.itemId.isBackendMinted:
result.add BackendLocalMarker
type
ParsedSymName* = object
@@ -327,9 +356,24 @@ proc writeLoc(w: var Writer; dest: var TokenBuf; loc: TLoc) =
writeFlags(dest, loc.flags) # TLocFlags
dest.addStrLit loc.snippet
proc nifTypeName(w: Writer; typ: PType): string =
## NIF name of a type as written by THIS module. A process-local backend env
## type is re-homed to the current module with the `@bk` marker (see
## BackendLocalMarker); everything else uses the canonical `typeToNifSym`.
if typ.uniqueId.isBackendMinted:
result = "`t"
result.addInt ord(typ.kind)
result.add '.'
result.addInt typ.uniqueId.item
result.add '.'
result.add modname(w.currentModule, w.infos.config)
result.add BackendLocalMarker
else:
result = typeToNifSym(typ, w.infos.config)
proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) =
dest.buildTree tdefTag:
dest.addSymDef pool.syms.getOrIncl(typeToNifSym(typ, w.infos.config)), NoLineInfo
dest.addSymDef pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo
dest.addDotToken # always private for the index generator
#dest.addIdent toNifTag(typ.kind)
@@ -352,7 +396,13 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) =
writeType(w, dest, typ.typeInstImpl)
#if typ.kind in {tyProc, tyIterator} and typ.nImpl != nil and typ.nImpl.kind != nkFormalParams:
# The reclist holds this type's OWN fields. A type can be force-loaded by
# name in isolation (cg seeks the `.t.nif`/`.s.nif` index entry), so its
# fields must be DEFS here, not entry-deduped SymUses whose def lives
# elsewhere in the `(lowered)` entry and is never read by the seek.
inc w.inTypeReclist
writeNode(w, dest, typ.nImpl)
dec w.inTypeReclist
writeSym(w, dest, typ.ownerFieldImpl)
writeSym(w, dest, typ.symImpl)
@@ -367,6 +417,15 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) =
proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) =
if typ == nil:
dest.addDotToken()
elif typ.uniqueId.isBackendMinted:
# Process-local closure env (see transf.transformBody): emit a MODULE-LOCAL
# `@bk` def the first time it is reached in this module, reference it after.
# Per-Writer dedup (NOT the shared `state`), since every referencing module
# must emit its own copy.
if not w.emittedBackendTypes.containsOrIncl(typ.uniqueId.item):
writeTypeDef(w, dest, typ)
else:
dest.addSymUse pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo
elif typ.uniqueId.module == w.currentModule and typ.state == Complete:
# Ownership for serialization is decided by `uniqueId`, not `itemId`: the NIF
# name (`typeToNifSym`) and the loader (`createTypeStub`) both key off
@@ -379,7 +438,7 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) =
if w.infos.config.ideActive: w.writtenTypes.add typ
writeTypeDef(w, dest, typ)
else:
dest.addSymUse pool.syms.getOrIncl(typeToNifSym(typ, w.infos.config)), NoLineInfo
dest.addSymUse pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo
proc writeBool(dest: var TokenBuf; b: bool) =
dest.buildTree (if b: "true" else: "false"):
@@ -481,6 +540,18 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) =
writeLoc w, dest, sym.locImpl
writeNode(w, dest, sym.constraintImpl)
writeSym(w, dest, sym.instantiatedFromImpl)
# The TRANSFORMED body (ic_ideas.md 2-way body): a routine run at compile time
# (macro / VM transform / `static`) already has its lowered body — closure
# `:env` and all — computed during sem; serialize it so the backend reuses it
# instead of re-deriving (the divergence behind the t17.275 env class). An
# empty `.` here means "same as the semchecked body OR to be found in the
# `.t.nif`" (the `lower` stage fills that gap). Non-routines / not-yet-
# transformed routines write the empty marker. (`transformedBodyImpl` only
# exists in the routine branch of the `TSym` variant.)
if sym.kindImpl in routineKinds:
writeNode(w, dest, sym.transformedBodyImpl)
else:
dest.addDotToken
dest.addParRi
@@ -501,9 +572,31 @@ proc shouldWriteSymDef(w: var Writer; sym: PSym): bool {.inline.} =
return true # Normal case for global symbols
return false
proc isLoweredPerEntryField(w: Writer; sym: PSym): bool {.inline.} =
## In the `lower` stage every entry (each `(lowered)` body AND each `@bk` type
## def, which carries its fields inline) is loaded independently, so it must be
## SELF-CONTAINED. A derived closure-env FIELD is module-homed (its id derives
## from the captured local, NOT `@bk` — see itemids.derivedFieldId) so
## `shouldWriteSymDef` would seal it after the first entry and later entries
## would reference a def that their indexed copy does not contain. Re-emit it
## as a full def per entry, deduped within the entry via `emittedFieldSyms`.
w.lowering and sym.kindImpl == skField and not sym.itemId.isBackendMinted
proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) =
if sym == nil:
dest.addDotToken()
elif isLoweredPerEntryField(w, sym):
if not w.emittedFieldSyms.containsOrIncl(sym.itemId):
writeSymDef(w, dest, sym)
else:
dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo
elif sym.itemId.isBackendMinted:
# Process-local backend sym (closure env field / hidden `:env` param): emit a
# MODULE-LOCAL `@bk` def the first time, reference it after. Per-Writer dedup.
if not w.emittedBackendSyms.containsOrIncl(sym.itemId.item):
writeSymDef(w, dest, sym)
else:
dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo
elif shouldWriteSymDef(w, sym):
sym.state = Sealed
if w.infos.config.ideActive: w.writtenSyms.add sym
@@ -529,8 +622,23 @@ proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) =
var nodeTyp = n.typField
if nodeTyp == nil and nfLazyType in n.flags:
nodeTyp = sym.typImpl
if shouldWriteSymDef(w, sym):
sym.state = Sealed
# Backend-minted syms (process-local closure `:env` param/fields) are emitted
# as MODULE-LOCAL `@bk` defs the first time reached this module (per-Writer
# dedup shared with `writeSym`), regardless of module: their itemId.module is
# the systemModule of `vmTransfIdgen`, so `shouldWriteSymDef` (which gates on
# currentModule) would otherwise only ever emit a SymUse → dangling def.
let perEntryField = isLoweredPerEntryField(w, sym)
# A field reached while writing its own type's reclist MUST be a self-contained
# def: the type can be seek-loaded by name in isolation, so a deduped SymUse
# (whose def lives elsewhere in the entry) would resolve to nil.
let reclistField = w.lowering and w.inTypeReclist > 0 and sym.kindImpl == skField
let wantDef =
if reclistField: true
elif sym.itemId.isBackendMinted: not w.emittedBackendSyms.containsOrIncl(sym.itemId.item)
elif perEntryField: not w.emittedFieldSyms.containsOrIncl(sym.itemId)
else: shouldWriteSymDef(w, sym)
if wantDef:
if not sym.itemId.isBackendMinted and not perEntryField and not reclistField: sym.state = Sealed
if w.infos.config.ideActive: w.writtenSyms.add sym
if nodeTyp != n.sym.typImpl:
dest.buildTree hiddenTypeTag, trLineInfo(w, n.info):
@@ -644,6 +752,7 @@ var importTag = registerTag("import")
var implTag = registerTag("implementation")
var reexpModTag = registerTag("reexpmod")
var offerTag = registerTag("offer")
var typeOfferTag = registerTag("toffer")
var modulesrcTag = registerTag("modulesrc")
proc registerNifAstTags*() =
@@ -676,6 +785,7 @@ proc registerNifAstTags*() =
implTag = registerTag("implementation")
reexpModTag = registerTag("reexpmod")
offerTag = registerTag("offer")
typeOfferTag = registerTag("toffer")
modulesrcTag = registerTag("modulesrc")
proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) =
@@ -1293,6 +1403,7 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
genericOffers: seq[tuple[generic, inst: PSym;
concreteTypes: seq[PType];
genericParamsCount: int]] = @[];
typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[];
resolvedImportDeps: seq[FileIndex] = @[]) =
var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule)
var content = createTokenBuf(300)
@@ -1370,6 +1481,47 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
w.deps.addStrLit toFullPath(config, FileIndex(thisModule))
w.deps.addParRi
# Generic TYPE-instance OFFERS: the `tyGenericInst` types this module created
# (e.g. `HashArray[8192, Gwei]`). Non-IC keeps ONE such instance in the global
# `typeInstCache`, so a structural bound computed at the first instantiation
# site (e.g. an `array[…]` bound that depends on a `mixin`/`compiles()` whose
# resolution differs by import scope) is frozen and reused everywhere. A
# separate `nim m` process never repopulates `typeInstCache` from NIFs, so it
# re-instantiates in its own scope and can compute a DIFFERENT bound (the SSZ
# `dataPerChunk` divergence). The loader rebuilds `g.typeInstCache` from these
# so `semtypinst.searchInstTypes` hits and reuses the baked instance.
# Layout: (toffer <genericBodySym> <instType>).
for off in typeOffers:
# Carry the generic body sym and the instance type as STRING LITERALS, not
# SymUse tokens: `addSymUse` rewrites a same-module reference into the NIF
# "local form" (suffix stripped, resolved by the content loader against the
# module being read), but this offer lives in the `deps` header and is read
# by a CONSUMER with no such module context. The full names round-trip
# verbatim as strings and `createTypeStub`/`resolveHookSym` resolve them
# directly (cf. `loadImport`, which carries module suffixes the same way).
w.deps.addParLe typeOfferTag, NoLineInfo
w.deps.addStrLit w.toNifSymName(off.generic)
w.deps.addStrLit typeToNifSym(off.inst, w.infos.config)
w.deps.addParRi
# OWNER MUST EMIT: a type reachable only through an offered instance — the
# `concreteTypes` of an offered proc instance (e.g. chronicles `writeValue[T]`,
# where `T` is this module's own object type) or an offered generic type
# instance — may never be reached by the normal top-level serialization above.
# If this module OWNS such a type, force-emit its typedef so that a consumer
# which reuses the offer can resolve the cross-module SymUse to it. Without this
# the consumer writes `t<k>.<i>.<thisSuffix>` and the loader asserts
# `symbol has no offset`. `writeType` emits the def (and recurses into owned
# sons) only for an own, still-Complete type; an already-Sealed one is skipped.
for off in genericOffers:
for ct in off.concreteTypes:
if ct != nil and ct.uniqueId.module == w.currentModule and ct.state == Complete:
writeType(w, bottom, ct)
for off in typeOffers:
if off.inst != nil and off.inst.uniqueId.module == w.currentModule and
off.inst.state == Complete:
writeType(w, bottom, off.inst)
# the implTag is used to tell the loader that the
# bottom of the file is the implementation of the module:
content.addParLe implTag, NoLineInfo
@@ -1378,7 +1530,7 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
content.addParRi()
let m = modname(w.currentModule, w.infos.config)
let nifFilename = AbsoluteFile(m).changeFileExt(".nif")
let nifFilename = AbsoluteFile(m).changeFileExt(".s.nif")
let d = completeGeneratedFilePath(config, nifFilename).string
var dest = createTokenBuf(600)
@@ -1586,7 +1738,7 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}):
# but haven't had their NIF index loaded yet
let hasEntry = c.mods.hasKey(result)
if not hasEntry or AlwaysLoadInterface in flags:
let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string
let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.nif")).string
if not fileExists(modFile):
raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile &
". This can happen when loading a module from NIF that references another module " &
@@ -1644,7 +1796,10 @@ proc tryCreateTypeStub(c: var DecodeContext; t: SymId): PType =
let suffix = name.substr(i)
if suffix == SysModuleSuffix:
return reconstructSysType(c, name, k, itemVal)
let id = itemId(moduleId(c, suffix).int32, itemVal)
let isBk = suffix.endsWith(BackendLocalMarker)
let realSuffix = if isBk: suffix[0 ..< suffix.len - BackendLocalMarker.len] else: suffix
let modIdx = moduleId(c, realSuffix).int32
let id = if isBk: backendItemId(modIdx, itemVal) else: itemId(modIdx, itemVal)
let ii = addr c.mods[id.module.FileIndex].index
let offs = ii[].getOrDefault(name)
if offs.offset == 0:
@@ -1671,9 +1826,13 @@ proc createTypeStub(c: var DecodeContext; t: SymId): PType =
let suffix = name.substr(i)
if suffix == SysModuleSuffix:
return reconstructSysType(c, name, k, itemVal)
let id = itemId(moduleId(c, suffix).int32, itemVal)
let ii = addr c.mods[id.module.FileIndex].index
let offs = ii[].getOrDefault(name)
let isBk = suffix.endsWith(BackendLocalMarker)
let realSuffix = if isBk: suffix[0 ..< suffix.len - BackendLocalMarker.len] else: suffix
let modIdx = moduleId(c, realSuffix).int32
let id = if isBk: backendItemId(modIdx, itemVal) else: itemId(modIdx, itemVal)
let modFi = id.module.FileIndex
let ii = addr c.mods[modFi].index
var offs = ii[].getOrDefault(name)
if offs.offset == 0:
raiseAssert "symbol has no offset: " & name
result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial)
@@ -1759,10 +1918,16 @@ proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string;
# Global symbol - look up in index for lazy loading
result = c.syms.getOrDefault(symAsStr)[0]
if result == nil:
let module = moduleId(c, sn.module)
# A process-local backend sym (closure env field / `:env` param) is named
# `…<thisModuleSuffix>@bk`: home it to that module with a backendItemId so it
# stays disjoint from the loader's real per-module id space (see toNifSymName).
let isBk = sn.module.endsWith(BackendLocalMarker)
let realMod = if isBk: sn.module[0 ..< sn.module.len - BackendLocalMarker.len]
else: sn.module
let module = moduleId(c, realMod)
let val = addr c.mods[module].symCounter
inc val[]
let id = itemId(module.int32, val[])
let id = if isBk: backendItemId(module.int32, val[]) else: itemId(module.int32, val[])
let offs = c.mods[module].index.getOrDefault(symAsStr)
if offs.offset == 0:
@@ -1843,7 +2008,12 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms
raiseAssert "(td) expected"
var scanCursor = n # copy cursor at start of type
let typesModule = parseSymName(pool.syms[n.firstSon.symId]).module
var typesModule = parseSymName(pool.syms[n.firstSon.symId]).module
if typesModule.endsWith(BackendLocalMarker):
# A backend-minted (`@bk`) type's name carries the marker in its module part;
# strip it so the nested-local pre-scan resolves the real module, not a
# nonexistent `<suffix>@bk.nif`.
typesModule = typesModule[0 ..< typesModule.len - BackendLocalMarker.len]
extractLocalSymsFromTree(c, scanCursor, typesModule, localSyms)
inc n # move past (td
@@ -1882,8 +2052,18 @@ proc loadType*(c: var DecodeContext; t: PType) =
if t.state != Partial: return
t.state = c.loadedState
var buf = createTokenBuf(30)
let typeName = typeToNifSym(t, c.infos.config)
var n = cursorFromIndexEntry(c, t.itemId.module.FileIndex, c.types[typeName][1], buf)
# A backend-minted (`@bk`) closure-env type produced by the `lower` stage lives
# ONLY in the `.t.nif` and is keyed by its `@bk` name (see nifTypeName), not the
# canonical `typeToNifSym` (which asserts non-`@bk`). Reconstruct that name so a
# Partial `@bk` stub that escaped the inline pre-scan can still be force-loaded.
let typeName =
if t.uniqueId.isBackendMinted:
"`t" & $ord(t.kind) & "." & $t.uniqueId.item & "." &
modname(t.itemId.module, c.infos.config) & BackendLocalMarker
else:
typeToNifSym(t, c.infos.config)
let modFi = t.itemId.module.FileIndex
var n = cursorFromIndexEntry(c, modFi, c.types[typeName][1], buf)
var localSyms = initTable[string, PSym]()
loadTypeFromCursor(c, n, t, localSyms)
@@ -1968,6 +2148,15 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule:
loadLoc c, n, s.locImpl
s.constraintImpl = loadNode(c, n, thisModule, localSyms)
s.instantiatedFromImpl = loadSymStub(c, n, thisModule, localSyms)
# The TRANSFORMED body slot (see writeSymDef). Reconstruct it ONLY in the
# backend (`cmdNifC`), where `transformBody` short-circuits on it; during
# frontend sem (`cmdM`) skip the tokens — a dependent never needs a foreign
# routine's lowered body, and reconstructing one must not perturb effect/
# exception inference (the "never change frontend node-typing for IC" rule).
if c.infos.config.cmd == cmdNifC and s.kindImpl in routineKinds:
s.transformedBodyImpl = loadNode(c, n, thisModule, localSyms)
else:
skip n
skipParRi n
proc loadSym*(c: var DecodeContext; s: PSym) =
@@ -2000,6 +2189,21 @@ proc loadSym*(c: var DecodeContext; s: PSym) =
if docId != 0'u32 and s.astImpl != nil and nodeCommentWriter != nil:
nodeCommentWriter(s.astImpl, pool.strings[StrId(docId)])
proc sealLoadedRoutines*(c: var DecodeContext) =
## Before `writeLoweredModule` re-serializes the lowered module, seal ONLY the
## module's ROUTINE syms. A `.t.nif` written by `writeLoweredModule` is the
## SOLE source the `cg` stage loads (there is no `.s.nif` fallback for its
## bodies), so every type, global, param and local must still emit a REAL def
## in it — only cross-routine references may be `SymUse`s (each routine's def is
## emitted once, at module scope, by the explicit stub loop). Types/globals stay
## `Complete` so `writeType`/`writeGlobals` emit them; routines become `Sealed`
## so a body referencing another routine writes a `SymUse` resolved through the
## module index.
for _, v in c.syms:
if v[0] != nil and v[0].state == Complete and v[0].kindImpl in routineKinds:
v[0].state = Sealed
proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym
template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) =
let info = c.infos.oldLineInfo(n.info)
@@ -2073,9 +2277,46 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string;
loadSymFromCursor(c, sym, n, thisModule, localSyms)
sym.state = c.loadedState # mark as fully loaded
result = newSymNode(sym, info)
else:
elif sn.module.endsWith(BackendLocalMarker):
# A backend-minted (`@bk`) def lives ONLY inline in this `.t.nif` body
# (not in any module index): create/find its cached stub and FILL it
# from the sdef instead of skipping (which would leave the skModule/
# Partial stub `loadSymStub` made unresolved).
sym = c.loadSymStub(name.symId, thisModule, localSyms)
skip n # skip the entire sdef for indexed symbols
if sym.state == Partial:
sym.state = c.loadedState
inc n # skip `sd` tag
loadSymFromCursor(c, sym, n, thisModule, localSyms)
else:
skip n
result = newSymNode(sym, info)
result.flags.incl nfLazyType
else:
# A module-homed inline sdef. Normally its def lives in that module's
# index and is loaded lazily, so we skip the inline copy. BUT a
# transform-created closure-env FIELD (`x0.0.clo`) is module-homed yet
# lives ONLY inline in this `.t.nif` reclist — it has no index entry.
# Skipping it leaves a nil-typed `skModule` fallback stub (from
# loadSymStub's "no offset" path) and codegen of the env struct then
# dereferences a nil field type. Detect the unindexed case and FILL
# the sym from the inline def instead.
let m = moduleId(c, sn.module)
let indexed = c.mods[m].index.hasKey(symName)
if indexed:
sym = c.loadSymStub(name.symId, thisModule, localSyms)
skip n # skip the entire sdef for indexed symbols
else:
sym = c.syms.getOrDefault(symName)[0]
if sym == nil:
let val = addr c.mods[m].symCounter
inc val[]
sym = PSym(itemId: itemId(m.int32, val[]), kindImpl: skStub,
name: c.cache.getIdent(sn.name), disamb: sn.count.int32,
state: Partial)
c.syms[symName] = (sym, NifIndexEntry())
sym.state = c.loadedState
inc n # skip `sd` tag
loadSymFromCursor(c, sym, n, thisModule, localSyms)
result = newSymNode(sym, info)
result.flags.incl nfLazyType
of typeDefTagName:
@@ -2166,11 +2407,14 @@ proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex;
if result == nil:
let symAsStr = nifName
let sn = parseSymName(symAsStr)
let symModule = moduleId(c, if sn.module.len > 0: sn.module else: thisModule)
let rawMod = if sn.module.len > 0: sn.module else: thisModule
let isBk = rawMod.endsWith(BackendLocalMarker)
let realMod = if isBk: rawMod[0 ..< rawMod.len - BackendLocalMarker.len] else: rawMod
let symModule = moduleId(c, realMod)
let val = addr c.mods[symModule].symCounter
inc val[]
let id = itemId(symModule.int32, val[])
let id = if isBk: backendItemId(symModule.int32, val[]) else: itemId(symModule.int32, val[])
let (stubKind, stubName) = stubKindAndName(c.cache, sn.name)
result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial)
c.syms[symAsStr] = (result, entry)
@@ -2233,7 +2477,15 @@ proc moduleSymbolStubs*(c: var DecodeContext; module: FileIndex): seq[PSym] =
proc toNifFilename*(conf: ConfigRef; f: FileIndex): string =
let suffix = moduleSuffix(conf, f)
result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string
# The `cg`/`emit` backend stages load the lowered whole-module NIF (transformed
# bodies + lifted sigs baked in); the `lower` stage and the frontend (`cmdM`)
# read the semchecked `.s.nif`.
if conf.cmd == cmdNifC and
(conf.icBackendStage == "cg" or conf.icBackendStage == "emit"):
let t = toGeneratedFile(conf, AbsoluteFile(suffix), ".t.nif").string
if fileExists(t):
return t
result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.nif").string
proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym =
result = c.syms.getOrDefault(symAsStr)[0]
@@ -2243,7 +2495,10 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo
let sn = parseSymName(symAsStr)
if sn.module.len == 0:
return nil # Local symbols shouldn't be hooks
let module = moduleId(c, sn.module)
let isBk = sn.module.endsWith(BackendLocalMarker)
let realMod = if isBk: sn.module[0 ..< sn.module.len - BackendLocalMarker.len]
else: sn.module
let module = moduleId(c, realMod)
# Look up the symbol in the module's index
# Try both formats: with module suffix (e.g., "foo.0.modulename") and without (e.g., "foo.0.")
# NIF spec allows local symbols to be stored without module suffix
@@ -2259,7 +2514,7 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo
# Create a stub symbol
let val = addr c.mods[module].symCounter
inc val[]
let id = itemId(int32(module), val[])
let id = if isBk: backendItemId(int32(module), val[]) else: itemId(int32(module), val[])
result = PSym(itemId: id, kindImpl: skProc, name: c.cache.getIdent(sn.name),
disamb: sn.count.int32, state: Partial)
c.syms[symAsStr] = (result, offs)
@@ -2356,6 +2611,11 @@ type
## generic instances this module created; modulegraphs.nim rebuilds
## `procInstCache` from them so a consumer reuses the instance instead of
## re-instantiating it in its own (operator-blind) module scope.
typeOffers*: seq[tuple[generic: PSym; inst: PType]]
## generic TYPE instances this module created; modulegraphs.nim rebuilds
## `typeInstCache` from them so a consumer reuses the baked instance
## (e.g. a `mixin`/`compiles()`-dependent array bound) instead of
## re-instantiating it with a different bound in its own scope.
includes*: seq[string] # resolved full paths of files this module `include`s;
# replayed into `inclToMod` by modulegraphs.nim so that
# nimsuggest can map a query in an include file back to
@@ -2625,6 +2885,35 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag];
t = next(s)
if ok and genSym != nil and instSym != nil:
result.genericOffers.add (genSym, instSym, cts, paramsCount)
elif t.tagId == typeOfferTag:
# (toffer "<genericBodySym>" "<instType>") — see the writer. The two
# full names arrive as string literals; intern them and resolve to a
# PSym/PType, then FULLY load the instance (its array bounds/fields) so
# `searchInstTypes` can match its params (a Partial stub has empty kids).
# Best-effort: a type that fails to resolve drops the offer.
t = next(s) # skip (toffer
var genName, instName = ""
var idx = 0
while t.kind != ParRi and t.kind != EofToken:
if t.kind == StringLit:
if idx == 0: genName = pool.strings[t.litId]
elif idx == 1: instName = pool.strings[t.litId]
inc idx
t = next(s)
if t.kind != ParRi:
raiseAssert "expected ParRi in toffer entry of module " & suffix
t = next(s)
if genName.len > 0 and instName.len > 0:
# Resolving/loading these entities lazily reads from the SAME stream we
# are iterating, moving its cursor — save/restore around it (cf. the
# export-list handling above).
let saved = offset(s.r)
let genSym = resolveHookSym(c, pool.syms.getOrIncl(genName))
let inst = tryCreateTypeStub(c, pool.syms.getOrIncl(instName))
if genSym != nil and inst != nil:
loadType(c, inst)
result.typeOffers.add (genSym, inst)
s.r.jumpTo(saved)
elif t.tagId == modulesrcTag:
# self-identification record for the standalone include-graph scanner;
# not needed by the lazy loader, just skip past it.
@@ -2675,6 +2964,112 @@ proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: va
let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f))
result = loadNifModule(c, suffix, interf, interfHidden, flags)
proc writeLoweredModule*(c: var DecodeContext; config: ConfigRef;
precomp: PrecompiledModule;
hooks: openArray[LogEntry]; outfile: string) =
## Re-serialize a backend-loaded module as a FULL module NIF (`.t.nif`) whose
## routine `(sd)` entries carry their TRANSFORMED bodies (the `lower` stage set
## them, recursively lifting nested closures — including the async state-machine
## procs whose inner closure the per-`(lowered)`-entry path failed to cross) and
## whose lambda-lift-minted entities (closure-env types/syms, lifted nested
## procs) are real, indexed defs. The `cg` stage then loads it through the
## normal module loader (`moduleFromNifFile`), so a transformed body arrives via
## `loadSymFromCursor`'s Step-A 2-way-body slot WITH the lifted signature — no
## `(lowered)` side-car, no `:envP` re-weld. This realizes `ic_ideas.md`'s eager
## two-way body whole-module.
let thisModule = precomp.module.positionImpl.int32
# Routines → Sealed (cross-routine refs become SymUse, defs emitted once below);
# types/globals/params/locals stay Complete and emit real defs (the `.t.nif` is
# the sole source the cg stage reads — no `.s.nif` fallback for them).
sealLoadedRoutines(c)
var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule)
w.inProc = 1
w.lowering = true
var content = createTokenBuf(300)
let rootInfo = trLineInfo(w, precomp.topLevel.info)
createStmtList(content, rootInfo)
# This module's ops (hooks/converters/methods/pure-enums) loaded from `.s.nif`,
# plus the type-bound ops the lower transform just lifted (closure-env
# `=destroy` etc., which have no `.s.nif` entry).
for op in precomp.logOps:
if op.module == thisModule.int:
writeOp(w, content, op)
for op in hooks:
writeOp(w, content, op)
var bottom = createTokenBuf(300)
# Imperative init code + global let/var/const sections + replay actions — all
# that a backend-loaded `topLevel` carries (routines are lazy index sdefs, not
# here). Emits + seals the module's globals.
w.writeToplevelNode content, bottom, precomp.topLevel
# Routine DEFS with transformed bodies, sourced from the index.
for s in moduleSymbolStubs(c, FileIndex thisModule):
if s.kindImpl in routineKinds and s.itemId.module == thisModule:
writeSymDef(w, bottom, s)
# Lifted hook ROUTINES (`@bk`, NEW in the lower stage — no `.s.nif` sdef, so
# absent from `moduleSymbolStubs`): emit each as a full def (sig + transformed
# body) so `injectDestructorCalls` in cg resolves the loaded env's `=destroy`.
var emittedHooks = initHashSet[int32]()
for op in hooks:
if op.sym != nil and op.sym.kindImpl in routineKinds and
not emittedHooks.containsOrIncl(op.sym.itemId.item):
writeSymDef(w, bottom, op.sym)
# deps / reexports / offers — mirror writeNifModule so the cg backend closure
# walk, interface re-export and generic-instance reuse all work off `.t.nif`.
for dep in precomp.deps:
if not w.depSuffixes.containsOrIncl(dep.string):
w.deps.addParLe importTag, NoLineInfo
w.deps.addDotToken
w.deps.addDotToken
w.deps.addStrLit dep.string
w.deps.addParRi
for (mname, msuffix) in precomp.reexportedModules:
w.deps.addParLe reexpModTag, NoLineInfo
w.deps.addStrLit mname
w.deps.addStrLit msuffix
w.deps.addParRi
for off in precomp.genericOffers:
w.deps.addParLe offerTag, NoLineInfo
w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.generic)), NoLineInfo
w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.inst)), NoLineInfo
w.deps.addIntLit off.genericParamsCount
for ct in off.concreteTypes:
w.deps.addSymUse pool.syms.getOrIncl(typeToNifSym(ct, w.infos.config)), NoLineInfo
w.deps.addParRi
for off in precomp.typeOffers:
w.deps.addParLe typeOfferTag, NoLineInfo
w.deps.addStrLit w.toNifSymName(off.generic)
w.deps.addStrLit typeToNifSym(off.inst, w.infos.config)
w.deps.addParRi
# OWNER MUST EMIT offered types this module owns (see writeNifModule).
for off in precomp.genericOffers:
for ct in off.concreteTypes:
if ct != nil and ct.uniqueId.module == w.currentModule and ct.state == Complete:
writeType(w, bottom, ct)
for off in precomp.typeOffers:
if off.inst != nil and off.inst.uniqueId.module == w.currentModule and
off.inst.state == Complete:
writeType(w, bottom, off.inst)
# Assemble exactly as writeNifModule: (stmts . . <deps> <ops+toplevel>
# (implementation) <bottom> ).
content.addParLe implTag, NoLineInfo
content.addParRi()
content.add bottom
content.addParRi()
var dest = createTokenBuf(600)
createStmtList(dest, rootInfo)
dest.add w.deps
for i in 3 ..< content.len-1:
dest.add content[i]
dest.addParRi()
writeFile(dest, outfile)
when isMainModule:
import std / syncio
let obj = parseSymName("a.123.sys")

17
compiler/astdef.nim
View File

@@ -17,6 +17,15 @@ when defined(nimPreviewSlimSystem):
export int128
var nifcBackendActive* = false
## Set only while the per-module NIF backend codegen stage runs
## (`nifbackend.generateCgStage`, `cmd == cmdNifC`). It gates `newSymNode`'s
## lazy-type marking so it applies ONLY in the backend — where syms are loaded
## from NIF and a cg-stage transform can build a sym node from a not-yet-typed
## stub — and never during frontend sem, where the same marking would perturb
## effect/exception inference (it diverges from a non-IC build, e.g.
## `times.toDateTimeByWeek` gaining a spurious unlisted `Exception`).
import nodekinds
export nodekinds
@@ -970,6 +979,14 @@ proc newSymNode*(sym: PSym, info: TLineInfo): PNode =
result = newNode(nkSym)
result.sym = sym
result.typField = sym.typImpl
if result.typField == nil and nifcBackendActive:
# In the per-module NIF backend cg stage a transform (chronos async
# closure-iterator lowering) builds `result = …` sym nodes from a not-yet-typed
# NIF stub; snapshotting the nil here would leave the node permanently typeless
# and the backend later reads `t.flags` off it and SIGSEGVs (injectdestructors
# hasDestructor). Mark it lazy so `typ` re-reads `sym.typ` once resolved. Gated
# on `nifcBackendActive` so frontend sem is untouched (see the flag's doc).
result.flags.incl nfLazyType
result.info = info
proc newStrNode*(kind: TNodeKind, strVal: string): PNode =

29
compiler/ccgthreadvars.nim
View File

@@ -39,11 +39,30 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
if isExtern: Extern
elif lfExportLib in s.loc.flags: ExportLibVar
else: Private
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet,
typ = getTypeDesc(m, s.loc.t),
kind = Threadvar,
visibility = vis)
if m.config.cmd == cmdNifC and vis == Private and not isExtern:
# A `{.threadvar.}`/`{.global.}` thread-local declared inside a routine is
# emitted by every module that emit-everywhere's its enclosing routine
# (e.g. libp2p's `var keys {.global.}: HashSet`), so its content-addressed
# name collides at link. Same fix as a plain global (genGlobalVarDecl):
# `extern` declaration + a droppable `'d'` definition unit the merge stage
# assigns one owner. The thread-local storage class rides on both.
let cname = stripCnifMarks(s.loc.snippet)
let td = getTypeDesc(m, s.loc.t)
# `extern` declaration via the full `addVar` overload — it knows the
# thread-local storage class (`NIM_THREADVAR`); the simple `addVar`'s
# `addVarHeader` does not implement `Threadvar`.
m.s[cfsVars].addVar(m, s, name = s.loc.snippet, typ = td,
kind = Threadvar, visibility = Extern)
m.s[cfsVars].add(cnifDefDirective(cname, "d", icNifName(m, s)))
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet, typ = td, kind = Threadvar, visibility = vis)
m.s[cfsVars].add(cnifEndDefs())
else:
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet,
typ = getTypeDesc(m, s.loc.t),
kind = Threadvar,
visibility = vis)
proc generateThreadLocalStorage(m: BModule) =
if m.g.nimtv.buf.len != 0 and (usesThreadVars in m.flags or sfMainModule in m.module.flags):

58
compiler/ccgtypes.nim
View File

@@ -108,7 +108,14 @@ proc fillBackendName(m: BModule; s: PSym) =
var result: Rope
if s.kind in routineKinds and {optCDebug, optItaniumMangle} * m.g.config.globalOptions == {optCDebug, optItaniumMangle} and
m.g.config.symbolFiles == disabledSf:
result = mangleProc(m, s, false).rope
# Under the per-module IC backend the bare-name uniqueness probe
# (`m.g.mangledPrcs`) only sees the routines of the CURRENT module, so the
# clean-vs-`makeUnique` decision is made independently per process: a
# method base mangles clean at its owner but loses the in-module race to
# its same-signature dispatcher elsewhere (clean `speak` defined twice ->
# "multiple definition"; demanders call `speak_u<n>` that nobody defines).
# Force the stable, disamb-based unique name so every process agrees.
result = mangleProc(m, s, makeUnique = m.config.cmd == cmdNifC).rope
else:
let shared = sharedInstanceCName(m, s)
if shared.len > 0:
@@ -1408,10 +1415,24 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
m.hcrCreateTypeInfosProc.addCast(typ = ptrType(CPointer)):
m.hcrCreateTypeInfosProc.add(cAddr(name))
else:
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
if m.config.cmd == cmdNifC:
# Emit-everywhere (see genTypeInfoV1's perModuleCg gate): every demanding
# `cg` process emits this type info's tentative definition. Declare it
# `extern` first (the data analogue of a proc prototype) so a TU whose copy
# the merge stage drops still has a valid declaration; wrap the definition
# as a droppable `'d'` unit the merge stage assigns to a single owner so
# exactly one external-linkage tentative definition survives (preserving
# the RTTI pointer identity refc relies on).
m.s[cfsStrData].addDeclWithVisibility(Extern):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
m.s[cfsStrData].add(cnifDefDirective(name, "d", icNifName(m, origType)))
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
m.s[cfsStrData].add(cnifEndDefs())
m.icDataDefs.add (name, icNifName(m, origType))
else:
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
proc genTypeInfoAux(m: BModule; typ, origType: PType, name: Rope;
info: TLineInfo) =
@@ -1504,8 +1525,25 @@ proc genObjectFields(m: BModule; typ, origType: PType, n: PNode, expr: Rope;
m.s[cfsTypeInit3].addFieldAssignment(expr, "name", makeCString(field.name.s))
m.s[cfsTypeInit3].addFieldAssignment(expr, "sons", cAddr(subscript(tmp, cIntValue(0))))
m.s[cfsTypeInit3].addFieldAssignment(expr, "len", L)
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
if m.config.cmd == cmdNifC:
# The discriminator table has a content-addressed name
# (`NimDT_<hashType>_<field>`) and is emitted by every module that demands
# this variant type's RTTI (emit-everywhere; RTTI has no single owner —
# emission is lazy and often skipped). Declare it `extern` + wrap the
# tentative definition as a droppable `'d'` unit so the merge stage keeps
# exactly one external-linkage definition (mirrors the `TNimType` var and
# consts); otherwise the identical name collides across modules at link.
m.s[cfsData].addDeclWithVisibility(Extern):
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
m.s[cfsData].add(cnifDefDirective(tmp, "d", ""))
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
m.s[cfsData].add(cnifEndDefs())
m.icDataDefs.add (tmp, "")
else:
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
for i in 1..<n.len:
var b = n[i] # branch
var tmp2 = getNimNode(m)
@@ -2124,7 +2162,15 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
return prefixTI(result)
var owner = t.skipTypes(typedescPtrs).itemId.module
if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
# In the per-module backend (`cg`) V1 RTTI is emit-everywhere like procs,
# consts and V2 type info: every demanding module emits the `'d'` definition
# (deduped to one owner by the merge stage). The owner-routing below would
# instead push the definition into the owner module's *unwritten* backend
# module (discarded in this process) and emit only an extern here, leaving the
# symbol undefined at link — the refc `NTI*` undefined-reference bug. (V2 got
# this gate in 8e0dd4bfb; V1, only reached under `--mm:refc`, was missed.)
let perModuleCg = m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"
if not perModuleCg and owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
dbgNti "extern:ownerRouted"
# make sure the type info is created in the owner module
discard genTypeInfoV1(m.g.mods[owner], origType, info)

15
compiler/ccgutils.nim
View File

@@ -114,8 +114,19 @@ proc makeUnique(m: BModule; s: PSym, name: string = ""): string =
result = if name == "": s.name.s else: name
# keep backend-minted ids out of the `_u` namespace; their item counter
# restarts at 0 and would collide with loaded symbols' ids
result.add(if s.itemId.isBackendMinted: "_c" else: "_u")
result.add $s.itemId.item
if s.itemId.isBackendMinted:
result.add "_c"
result.add $s.itemId.item
else:
result.add "_u"
# Mirror `mangleProcNameExt`: use the per-(module,name) `disamb`, NOT
# `itemId.item`. Under the per-module IC backend the same symbol is loaded
# from a NIF in many processes and `itemId.item` is a fresh, load-order
# dependent counter — so a method base would mangle to `_u1` in one module,
# `_u3` in another and clean at its owner, none of which link. `disamb` is
# assigned deterministically per (module, name) and is serialized, so every
# process that touches the symbol derives the identical C name.
result.add $s.disamb
# module suffix LAST (a strippable trailing token; see `mangleProcNameExt`)
result.add "__"
result.add m.g.graph.ifaces[s.itemId.module].uniqueName

70
compiler/cgen.nim
View File

@@ -125,10 +125,45 @@ proc emitsBodyInThisModule(m: BModule, prc: PSym): bool =
## Generic instances and synthesized hooks (`=destroy`, `$`, …) have no single
## owning-module top-level — they are minted on demand — so each demander emits
## them and the merge stage deduplicates by their content-addressed C name.
##
## A NESTED routine is not emitted on its own: it is lambda-lifted and emitted
## as part of its ENCLOSING routine's body, into the same TU. So the decision
## must follow the OUTERMOST enclosing routine (the one directly under the
## module — `skipGenericOwner` stops at a generic *instance*, not its
## originating generic), never the nested symbol's own identity. Otherwise a
## nested proc whose enclosing is a generic instance (content-addressed,
## emitted by every demander) — e.g. nim-serialization's per-field `readField`
## inside the `makeFieldReadersTable[R,W]` instance, whose address fills the
## returned table — is gated out (its own `itemId.module` is the minting module
## and its disamb is a plain counter), so the enclosing's lift degrades it to a
## prototype and its body lands in no TU → undefined at link.
if not (m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"):
return true
result = prc.itemId.module == m.module.position or
(prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32
# The symbol may ITSELF be content-addressed (a synthesized hook or a generic
# instance carries `Hook/InstanceDisambBit` on its OWN `disamb`): then it has no
# single owning module and every demander emits it (merge dedups by C name),
# regardless of what it is nested under. This must be checked on `prc` directly,
# not on `top`: a `=destroy`/`=sink` lifted while compiling some enclosing proc
# (e.g. system's `isZeroMemory` destroying a `ptr array`) has that PROC as its
# `skipGenericOwner`, so `top` walks up to a plain routine whose own disamb has
# no bit — gating the hook to that routine's owner module, which mints it
# on demand and emits it nowhere → undefined at link.
if (prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32:
return true
var top = prc
while top.skipGenericOwner != nil and top.skipGenericOwner.kind != skModule:
top = top.skipGenericOwner
result = top.itemId.module == m.module.position or
(top.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32 or
# An INLINE iterator has no standalone body — it is expanded at each
# call site — so it is materialized in every module that iterates over
# it, never in its owner. A proc nested in one (e.g. std/uri's
# `parseData` inside `iterator decodeQuery`) is lambda-lifted into each
# of those consumer TUs and must be emitted there (its stable
# owner-suffixed name + `'u'` flag let the merge stage keep one); gating
# it to the iterator's owner module leaves it in no TU → undefined.
(top.kind == skIterator and top.typ != nil and
top.typ.callConv != ccClosure)
proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc =
result = TLoc(k: k, storage: s, lode: lode,
@@ -776,12 +811,31 @@ proc genGlobalVarDecl(res: var Builder, p: BProc, n: PNode; td: Snippet;
typ = constType(typ)
if p.hcrOn:
typ = ptrType(typ)
res.addVar(p.module, s,
name = s.loc.snippet,
typ = typ,
visibility = vis,
initializer = initializer,
initializerKind = initializerKind)
if p.config.cmd == cmdNifC and vis == Private and sfImportc notin s.flags:
# A `{.global.}` var (e.g. chronos's per-call-site `var loc {.global.} =
# SrcLoc(...)`, or a gensym'd `var dummy`/`var topic` with no initializer)
# declared inside a routine is emitted by every module that emit-everywhere's
# its enclosing routine; its content-addressed name then collides at link.
# Declare it `extern` + wrap the definition as a droppable `'d'` unit so the
# merge stage keeps exactly one (like consts / TNimType / the NimDT
# discriminator tables / the threadvar path). This covers no-initializer
# globals too — they collide just the same. A module-level global has a
# single claimant → its sole emitter is the owner merge keeps.
let cname = stripCnifMarks(s.loc.snippet)
res.addDeclWithVisibility(Extern):
res.addVar(kind = Local, name = s.loc.snippet, typ = typ)
res.add(cnifDefDirective(cname, "d", icNifName(p.module, s)))
res.addVar(p.module, s,
name = s.loc.snippet, typ = typ, visibility = vis,
initializer = initializer, initializerKind = initializerKind)
res.add(cnifEndDefs())
else:
res.addVar(p.module, s,
name = s.loc.snippet,
typ = typ,
visibility = vis,
initializer = initializer,
initializerKind = initializerKind)
proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
let s = n.sym

12
compiler/cgmeth.nim
View File

@@ -160,7 +160,17 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
var witness: PSym = nil
if s.typ.firstParamType.owner.getModule != s.getModule and vtables in g.config.features and not
g.config.isDefined("nimInternalNonVtablesTesting"):
g.config.isDefined("nimInternalNonVtablesTesting") and sfFromGeneric notin s.flags:
# `sfFromGeneric` excepted: this is the same-module restriction for vtable
# slot placement, and it must be judged on the GENERIC method, not on an
# instance. The generic `method skip[T](x: Input[T])` never reaches here
# (`semMethodPrototype` registers generic methods via `addMethodToGeneric`,
# bypassing `methodDef`); only its instance `skip[string]` does, and that
# instance's first-param type `Input[string]` is owned by whichever module
# first instantiated it (`tparsecombnum`, which `import parsecomb`s and uses
# it), NOT by `Input[T]`'s defining module — so the comparison spuriously
# fails for a method that is perfectly legal at the generic level. (Concrete
# methods, `sfFromGeneric notin flags`, are still checked.)
localError(g.config, s.info, errGenerated, "method `" & s.name.s &
"` can be defined only in the same module with its type (" & s.typ.firstParamType.typeToString() & ")")
if sfImportc in s.flags:

150
compiler/deps.nim
View File

@@ -51,7 +51,7 @@ proc parsedFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".p.nif"
proc semmedFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".nif"
getNimcacheDir(c.config).string / f.modname & ".s.nif"
proc ifaceFile(c: DepContext; f: FilePair): string =
## Interface-cookie sidecar written by `nim m` (ast2nif.writeIfaceCookie,
@@ -701,6 +701,14 @@ proc computeForwardedArgs(c: DepContext): seq[string] =
# buckets (and rejects calls as ambiguous that multi-dispatch accepts)
if optMultiMethods in c.config.globalOptions:
result.add "--multimethods:on"
# Forward the debug-info switch: the cg children — not the driver — fill the
# backend C names, and `--debugger:native` selects the Itanium mangling
# scheme (ccgtypes.fillBackendName). A child without it would name routines
# with the plain `_u<disamb>` scheme while a sibling that read the project's
# config.nims (`--debugger:native`) used Itanium, so the same symbol's
# definition and cross-module references would disagree at link.
if optCDebug in c.config.globalOptions:
result.add "--debugger:native"
# the children compile each MODULE as their own project file, which makes
# that module's package the "main package" and unfilters foreign-package
# diagnostics — a vendored package's hintAsError/warningAsError promotions
@@ -918,6 +926,36 @@ proc backendCFile(c: DepContext; node: Node): string =
result = changeFileExt(completeCfilePath(c.config,
mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string
proc computeLiveBackendNodes(c: DepContext): seq[bool] =
## Which nodes the backend must code-generate: the closure reachable from the
## program roots (main + `system` + `--import`ed modules) via the REAL,
## post-sem import edges (`.s.deps`).
##
## The static `.deps` scan over-approximates: it cannot evaluate guards like
## `when defined(windows)` or const-aliased ones (`when useWinVersion`, with
## `const useWinVersion = defined(windows) or defined(nimdoc)`), so it keeps
## the dead branch's import. e.g. on Linux `nativesockets`'s static deps list
## `winlean`; the discovery fixpoint only ever *adds* edges, never prunes, so
## `winlean` stays a node and got a full `lower`/`cg`/`emit`/link pipeline.
## That is harmless for sem (an extra `nim m`) but fatal for codegen:
## `winlean`'s `importc, header: "winsock2.h"` decls emit
## `#include "winsock2.h"` into a C file that cannot compile off-Windows.
## Sem's resolved import set (`.s.deps`) is the real program graph — the
## non-IC compiler would never touch `winlean` here — so restrict the backend
## to it. (`.s.deps` is the same data the discovery loop trusts; it is written
## for every sem'd module, including grouped SCC members.)
result = newSeq[bool](c.nodes.len)
var stack: seq[int] = @[0] # main module
if c.systemNodeId >= 0: stack.add c.systemNodeId
for impId in c.implicitNodeIds: stack.add impId # every module imports these
while stack.len > 0:
let ni = stack.pop()
if ni < 0 or ni >= c.nodes.len or result[ni]: continue
result[ni] = true
for p in readSemDeps(c, c.nodes[ni].files[0]):
let idx = c.processedModules.getOrDefault(c.toPair(p).modname, -1)
if idx >= 0: stack.add idx
proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
## Per-module backend build file. One `nim_nifc` command template (the actual
## stage/module switches ride in each rule's `(args …)`), then the stages of
@@ -941,9 +979,42 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
# Per-node output paths.
var cnifFiles = newSeq[string](c.nodes.len)
var cFiles = newSeq[string](c.nodes.len)
var tFiles = newSeq[string](c.nodes.len)
# The `lower` stage writes a PROPER module NIF the cg/emit stages load via
# `toNifFilename` (a `.s.nif` sibling), so its `.t.nif` lives at the suffix base
# (mirroring `semmedFile`), not next to the throwaway `.c`.
for i, node in c.nodes:
cFiles[i] = backendCFile(c, node)
cnifFiles[i] = cFiles[i] & ".nif"
tFiles[i] = nimcache / node.files[0].modname & ".t.nif"
# Only code-generate modules the real program actually reaches; statically
# over-approximated nodes (e.g. `winlean` on Linux) are sem'd but not emitted.
let live = computeLiveBackendNodes(c)
# Drop a pruned node's stale backend artifacts: the `merge` stage globs
# `*.c.nif` off disk (not the build-file inputs) and the `link` stage scans
# the loaded closure's `.c`s, so a leftover `.c.nif`/`.c` from a run before
# this module became unreachable (a prior over-approximated build, or an edit
# that removed its last real importer) would still be merged/compiled —
# reintroducing exactly the off-platform `#include` this prune avoids.
var prunedStale = false
for i in 0 ..< c.nodes.len:
if not live[i]:
# `fileExists` before remove so we only force a merge recompute (below)
# when an artifact was actually present — i.e. a build where this module
# WAS emitted, not the steady state where it never is.
if fileExists(cnifFiles[i]) or fileExists(cFiles[i]): prunedStale = true
removeFile(cnifFiles[i])
removeFile(cFiles[i])
# The merge decision is a pure function of the set of `.c.nif`s present; if we
# just removed an over-approximated module's artifacts, a decision computed
# while they were present is stale — it can name a now-absent module as a
# symbol's owner (`asyncdispatch` owning `NTIdomain` here), leaving that symbol
# undefined at link. nifmake will not re-fire `merge` on its own: dropping an
# input makes no remaining input newer than the output. Delete the decision so
# the (now missing) output forces a recompute against the live `.c.nif` set.
if prunedStale:
removeFile(mergeFile)
var b = nifbuilder.open(result)
defer: b.close()
@@ -965,9 +1036,12 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.addStrLit a
b.addTree "args"
b.endTree()
b.addTree "input"
b.addIntLit 0
b.endTree()
# The project file is a fixed command ARGUMENT, not a tracked input: backend
# stages read NIFs (resolved by suffix), never the `.nim` source, so its
# content cannot change any artifact. Passing it as `(input 0)` made its mtime
# an input to every rule, so editing the main module's source re-fired the
# whole backend.
b.addStrLit mainNif
b.endTree()
template inputStr(s: string) =
@@ -979,21 +1053,50 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.addStrLit s
b.endTree()
# cg: one rule per module. Inputs are the project (slot 0) and every semmed
# NIF (so the whole program loads and the rule is ordered after the frontend);
# the main module additionally depends on every other `.c.nif` (init metas).
# lower: one rule per module. Transforms (eventually) the routines the module
# OWNS once, in the owner's id space, into `<module>.t.nif`, so the `cg` stage
# reads them instead of re-deriving (which makes a closure `:env`'s identity
# diverge across the parallel `cg` processes). Runs per module in parallel.
#
# Input is this module's OWN semmed NIF and nothing else. A module does NOT
# depend on its importers, so listing every semmed NIF (or even the import
# closure) was wrong: it made e.g. `strutils`'s rule depend on the `finish`
# that imports it. nifmake handles the indirect dependency for free — the
# frontend writes `.s.nif`s content-stably, so an interface change to a
# dependency re-sems (and re-emits the `.s.nif` of) every transitive importer;
# a module whose own `.s.nif` is unchanged genuinely needs no re-lowering.
for i, node in c.nodes:
if not live[i]: continue
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:lower"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr c.semmedFile(node.files[0])
outputStr tFiles[i]
b.endTree()
# cg: one rule per module. Input is this module's OWN `.t.nif`. cg DOES read
# its dependencies' `.t.nif`s at runtime (loadDepClosure), but ordering is
# guaranteed by nifmake's depth-barriered scheduler: every `lower` is depth 1
# (its `.s.nif` is a leaf) and every `cg` is depth 2, so all lowering finishes
# before any cg starts — no need to list the closure for ordering. For
# invalidation, a dependency's change reaches this module through its own
# `.t.nif` (own `.s.nif` re-sem -> own `lower`); a foreign body this module
# emit-everywhere'd but does not own is dropped by `emit` regardless, so a
# stale copy here is harmless. The main module additionally depends on every
# other `.c.nif` (it reads their init/datInit metas to wire up NimMain).
for i, node in c.nodes:
if not live[i]: continue
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:cg"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr mainNif
for n2 in c.nodes:
inputStr c.semmedFile(n2.files[0])
inputStr tFiles[i]
if node.id == 0:
for j in 0 ..< c.nodes.len:
if c.nodes[j].id != 0:
if c.nodes[j].id != 0 and live[j]:
inputStr cnifFiles[j]
outputStr cnifFiles[i]
b.endTree()
@@ -1003,19 +1106,24 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:merge"
inputStr mainNif
for cn in cnifFiles: inputStr cn
for i in 0 ..< c.nodes.len:
if live[i]: inputStr cnifFiles[i]
outputStr mergeFile
b.endTree()
# emit: render each module's `.c` from its `.c.nif` + the merge decision.
for i, node in c.nodes:
if not live[i]: continue
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:emit"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr mainNif
# Inputs: this module's OWN `.c.nif` and the global merge decision. emit also
# loads `.t.nif`s at runtime (getCFile/type resolution), but those are depth 1
# and emit is past the merge barrier, so they always exist — no need to list
# them. (emit still re-fires for every module whenever `merge` rewrites the
# decision file; making that incremental is a separate concern.)
inputStr cnifFiles[i]
inputStr mergeFile
outputStr cFiles[i]
@@ -1026,8 +1134,8 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:link"
inputStr mainNif
for cf in cFiles: inputStr cf
for i in 0 ..< c.nodes.len:
if live[i]: inputStr cFiles[i]
outputStr exeFile
b.endTree()
@@ -1150,8 +1258,14 @@ proc commandIc*(conf: ConfigRef) =
# each DAG depth via execProcesses (defaults to all cores). Cold builds are
# otherwise serial (one child at a time) and leave the machine idle. Opt out
# with `-d:icNoParallel` (e.g. for readable, non-interleaved child output
# when debugging a build).
let parallel = if isDefined(conf, "icNoParallel"): "" else: " --parallel"
# when debugging a build), or cap the concurrency with `-d:icJobs:N` — an
# uncapped fan-out across many cores can exhaust RAM on a large project
# (each `nim m`/`cg` child holds its own module graph), which nifmake's own
# `-j:N` exists to bound.
let parallel =
if isDefined(conf, "icNoParallel"): ""
elif isDefined(conf, "icJobs"): " --parallel:" & conf.symbols["icJobs"]
else: " --parallel"
# Phase 1 — frontend (nifler + `nim m`), run to a discovery fixpoint.
var rounds = 0

12
compiler/injectdestructors.nim
View File

@@ -245,6 +245,18 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode
let canon = c.graph.canonTypes.getOrDefault(h)
if canon != nil:
op = getAttachedOp(c.graph, canon, kind)
if op == nil or op.ast.isGenericRoutine:
# IC: injectDestructorCalls is demand-driven and runs HERE (cg), not in the
# `lower` stage, so a structural, env-agnostic op the lower stage never had
# reason to serialize — most often a closure PROC type's `=destroy`/`=sink`
# (which act on the `(ClP_0, ClE_0)` tuple, NOT the concrete env) — must be
# lifted on demand, exactly as the lazy path's cg does. This is safe now:
# closure-env identity resolves via `attachedOps[itemId]`/env-erased typeKey,
# env objects load complete, and atomicRefOp's type-erased path covers any
# still-incomplete env (so the lift never walks a nil field).
excl t.flagsImpl, tfCheckedForDestructor
createTypeBoundOps(c.graph, nil, t, dest.info, c.idgen)
op = getAttachedOp(c.graph, t, kind)
if op == nil:
#echo dest.typ.id
globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &

2
compiler/lambdalifting.nim
View File

@@ -176,7 +176,7 @@ proc closureParams(routine: PSym): PNode =
result = routine.typ.n
routine.ast[paramsPos] = result
proc addHiddenParam(routine: PSym, param: PSym) =
proc addHiddenParam*(routine: PSym, param: PSym) =
assert param.kind == skParam
var params = closureParams(routine)
# -1 is correct here as param.position is 0 based but we have at position 0

8
compiler/liftdestructors.nim
View File

@@ -821,13 +821,15 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags
# dynamic Acyclic refs need to use dyn decRef
let useStatic = isFinal(elemType)
let tmp =
if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}:
declareTempOf(c, body, x)
else:
x
if isFinal(elemType):
if useStatic:
addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr))
var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
alignOf.typ = getSysType(c.g, c.info, tyInt)
@@ -838,7 +840,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
var cond: PNode
if isCyclic:
if isFinal(elemType):
if useStatic:
let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
typInfo.typ = getSysType(c.g, c.info, tyPointer)
cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
@@ -873,7 +875,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
of attachedDeepCopy: assert(false, "cannot happen")
of attachedTrace:
if isCyclic:
if isFinal(elemType):
if useStatic:
let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
typInfo.typ = getSysType(c.g, c.info, tyPointer)
body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y)

65
compiler/lowerings.nim
View File

@@ -210,7 +210,62 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym =
if matchesDerivedFieldId(n.sym.itemId, id): result = n.sym
else: discard
proc lookupCapturedField(n: PNode, s: PSym): PSym =
## Find an env field that `addField` would have produced for the captured
## local `s`. Used as a fallback when the derived-itemId match fails because
## `s` is a macro-generated gensym whose process-local id diverges from the
## loaded env field's (see `addField`). `addField` always names a field
## `s.name & $field.position`, so that pair uniquely identifies the field for a
## local of this name without relying on the (unstable) item id.
result = nil
case n.kind
of nkRecList:
for i in 0..<n.len:
result = lookupCapturedField(n[i], s)
if result != nil: return
of nkRecCase:
if n[0].kind != nkSym: return
result = lookupCapturedField(n[0], s)
if result != nil: return
for i in 1..<n.len:
case n[i].kind
of nkOfBranch, nkElse:
result = lookupCapturedField(lastSon(n[i]), s)
if result != nil: return
else: discard
of nkSym:
if n.sym.kind == skField and n.sym.name.s == s.name.s & $n.sym.position:
result = n.sym
else: discard
proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym =
# Idempotent w.r.t. the captured symbol (mirrors `addUniqueField`): re-lifting
# a LOADED routine re-derives its transformed body (never serialized under IC)
# and re-captures the same locals, but the env object loaded from the NIF
# already carries their fields. Re-adding would duplicate the field and, worse,
# mutate a Sealed loaded type via `propagateToOwner` (the `t.state != Sealed`
# crash). Return the existing field instead.
let existing = lookupInRecord(obj.n, s.itemId)
if existing != nil:
return existing
# Re-lifting a LOADED routine during a VM transform (its transformed body is
# re-derived per process, never serialized) re-captures the same locals, but
# for a macro-generated gensym (e.g. libp2p `p2pProtocolBackendImpl`'s
# `msgVar`) its process-local id diverges from the one baked into the loaded
# env field, so the id match above misses. Reuse the existing same-named field
# rather than appending a divergent duplicate, which keeps the re-derived
# closure consistent (else a stale `:env` access reaches `cannotEval`).
# Confined to a loaded (Sealed) env: in a freshly built env ids are consistent,
# and two distinct same-named captures legitimately get distinct fields there.
if obj.state == Sealed:
let byName = lookupCapturedField(obj.n, s)
if byName != nil:
return byName
# Genuinely new field. Under IC the env may be a loaded Sealed type whose
# transform-time mutation is process-local (the body is discarded after the
# macro runs), so downgrade it to mutable instead of crashing on
# `t.state != Sealed` (mirrors `markAsClosure`).
unsealForTransform(obj)
# because of 'gensym' support, we have to mangle the name with its ID.
# This is hacky but the clean solution is much more complex than it looks.
var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
@@ -306,6 +361,16 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
assert t.kind == tyObject
result = lookupInRecord(t.n, v.itemId)
if result != nil: break
# A LOADED (Sealed) env object carries fields baked by the producer process;
# re-lifting a NIF-loaded routine in a consumer (e.g. a macro VM-evaluating an
# imported `p2pProtocolBackendImpl`) re-captures the same local under a
# divergent process-local id, so the derived-itemId match misses. Fall back to
# the name+position identity `addField` uses — SYMMETRIC with `addField`'s
# Sealed by-name reuse — so the access resolves the field `addField` produced
# instead of failing with `not part of closure object type`.
if t.state == Sealed:
result = lookupCapturedField(t.n, v)
if result != nil: break
t = t.baseClass
if t == nil: break
t = t.skipTypes(skipPtrs)

70
compiler/modulegraphs.nim
View File

@@ -146,11 +146,19 @@ type
cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API; IC: implemented
cacheCounters*: Table[string, BiggestInt] # IC: implemented
cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
transitiveReplayActions*: seq[PNode] # macro-cache replay actions collected from
# the transitive import closure of a NIF-loaded module (loadTransitiveHooks);
# the caller (pipelines) replays them so a dependency's macrocache state — e.g.
# nim-serialization's flavor registration — reaches a module that imports it
# only indirectly. Drained per moduleFromNifFile call.
pendingNifInit*: seq[tuple[module: PSym; topLevel: PNode]]
# EVERY module loaded from a NIF — whether a direct import (moduleFromNifFile)
# or only a dep-of-a-dep (loadTransitiveHooks) — is recorded here with its
# serialized top-level AST. The sem driver drains it once
# (pipelines.finalizeLoadedModules) and applies the module's VM-level load
# effects UNIFORMLY: macro-cache replay (std/macrocache put/inc/add/incl) and
# eager `{.compileTime.}` global init. This is the single place "what a loaded
# module does to global state" lives, so a transitively-reached module — which
# never passes through compilePipelineModule — gets the SAME treatment as a
# direct import instead of silently skipping it (its macrocache state would be
# lost; its CT globals would stay nil and a macro splicing one, e.g.
# chronicles' `chroniclesBlockName`, emits `break nil` / `nil == 0`). To add a
# new per-load VM effect, extend the drain — never a parallel buffer.
passes*: seq[TPass]
pipelinePass*: PipelinePass
onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
@@ -160,6 +168,9 @@ type
strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.}
compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
idgen*: IdGenerator
vmTransfIdgen*: IdGenerator # process-local backend idgen for closure envs
# minted while the VM compiles a routine body
# (inVMTransform); see lambdalifting / ast2nif @bk
operators*: Operators
cachedFiles*: StringTableRef
@@ -898,7 +909,7 @@ proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
assert result != nil
when not defined(nimKochBootstrap):
proc registerLoadedHooks(g: ModuleGraph; logOps: seq[LogEntry]) =
proc registerLoadedHooks*(g: ModuleGraph; logOps: seq[LogEntry]) =
let mainSuffix = getMainModuleSuffix(ast.program)
for x in logOps:
# A dependency's NIF may carry hooks whose syms belong to the module we
@@ -954,14 +965,33 @@ when not defined(nimKochBootstrap):
if not g.hookClosure.containsOrIncl(fileIdx.int):
let precomp = loadNifModule(ast.program, suffix, interf, interfHidden, {})
registerLoadedHooks(g, precomp.logOps)
# Collect the dependency's macro-cache replay actions (put/inc/add/incl)
# so the importer being compiled also sees macrocache state registered
# by a transitively-imported module. Pragma replay actions are a backend
# concern and are intentionally not collected here.
for n in precomp.topLevel:
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
g.transitiveReplayActions.add n
# Record this transitively-loaded module so the sem driver applies its
# VM-level load effects (macro-cache replay + `{.compileTime.}` global init)
# exactly as for a direct import — see `pendingNifInit`. A throwaway module
# symbol (same shape as moduleFromNifFile's) gives the drain an idgen/info
# context; it is not registered, so a later direct import still loads fully.
if g.config.cmd == cmdM:
let m = PSym(kindImpl: skModule, itemId: itemId(int32(fileIdx), 0'i32),
name: getIdent(g.cache, splitFile(toFullPath(g.config, fileIdx)).name),
infoImpl: newLineInfo(fileIdx, 1, 1), positionImpl: int(fileIdx))
setOwner(m, getPackage(g.config, g.cache, fileIdx))
g.pendingNifInit.add (m, precomp.topLevel)
# Rebuild generic TYPE- and PROC-instance offers across the WHOLE closure,
# not just direct imports (`moduleFromNifFile`). An instance is frozen at
# the FIRST module to create it (in a scope where its body's symbols
# resolve unambiguously); a consumer many imports away must REUSE it rather
# than re-instantiate in its own scope, which may resolve a body symbol
# differently — a divergent `compiles()`-dependent array bound (SSZ
# `HashArray[8192, Gwei]`, type offer), or an ambiguous unqualified ident
# leaked from an unrelated import (`fromRaw` -> `SkRawPublicKeySize` from
# both `secp` and `secp256k1`, proc offer). Direct-only rebuild left the
# deep offer invisible when the clean instance lives a transitive hop away.
for off in precomp.typeOffers:
g.typeInstCache.mgetOrPut(off.generic.itemId, @[]).add off.inst
for off in precomp.genericOffers:
g.procInstCache.mgetOrPut(off.generic.itemId, @[]).add PInstantiation(
sym: off.inst, concreteTypes: off.concreteTypes,
genericParamsCount: off.genericParamsCount, compilesId: 0)
for d in precomp.deps: stack.add d
proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym =
@@ -1053,6 +1083,14 @@ when not defined(nimKochBootstrap):
sym: off.inst, concreteTypes: off.concreteTypes,
genericParamsCount: off.genericParamsCount, compilesId: 0)
# Rebuild `typeInstCache` from this module's generic TYPE-instance OFFERS so a
# consumer's `searchInstTypes` reuses the baked instance (e.g. an SSZ
# `HashArray` whose array bound depends on import-scope-sensitive `compiles()`)
# rather than re-instantiating it with a divergent bound — see ast2nif's
# `(toffer …)`. Keyed by the generic body sym's itemId, as `searchInstTypes`.
for off in result.typeOffers:
g.typeInstCache.mgetOrPut(off.generic.itemId, @[]).add off.inst
# Mark module as cached
g.cachedMods.incl fileIdx.int
g.hookClosure.incl fileIdx.int
@@ -1084,6 +1122,10 @@ when not defined(nimKochBootstrap):
# walks the closure in nifbackend.loadModuleDependencies.)
if g.config.cmd == cmdM:
loadTransitiveHooks(g, result.deps)
# Record the directly-loaded module for the same VM-level load effects as its
# transitive deps (`pendingNifInit`). AFTER loadTransitiveHooks so the drain
# applies deps before the dependent (macro-cache order).
g.pendingNifInit.add (m, result.topLevel)
proc isModuleFile(g: ModuleGraph; fileIdx: FileIndex): bool =
let i = fileIdx.int32

364
compiler/nifbackend.nim
View File

@@ -26,6 +26,7 @@ import ast, options, lineinfos, modulegraphs, cgendata, cgen,
pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif, typekeys,
cnif
from cgmeth import generateIfMethodDispatchers
from transf import transformBody
import ic / replayer
proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex;
@@ -139,12 +140,57 @@ proc signatureHasMetaType(t: PType; depth: int = 0): bool =
# as meta and drop it from the owned-routine seeding -> undefined symbols
# at link (its only definer never emits it).
return false
if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyStatic, tyGenericParam,
if t.kind == tyStatic:
# A RESOLVED static value (the `256` in `MDigest[256]`, the `N` in
# `HashList[T, N]`, …) is carried as a `tyStatic` node inside the otherwise
# fully-concrete `tyGenericInst`, but it is NOT meta: the routine is a normal
# runtime routine the owner must emit. Only an UNRESOLVED `static T` parameter
# (no bound value, `t.n == nil`) is meta. Without this, every routine whose
# signature touches a `static`-parameterized generic instance (the bulk of
# the SSZ/`MDigest` API) is dropped from the owned-routine seeding and ends up
# an undefined reference at link (mirrors the tyGenericBody case above).
return t.n == nil
if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyGenericParam,
tyAnything, tyFromExpr, tyError}:
return true
for k in t.kids:
if signatureHasMetaType(k, depth + 1): return true
proc ownsRuntimeRoutine(s: PSym; modPos: int): bool =
## A concrete, non-generic, runtime routine with a real body, OWNED by the
## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a
## routine called only from other modules is still emitted by somebody) and
## the `lower` stage's owned-routine enumeration, so both stages see exactly
## the same set. The exclusions:
## - nested/closure procs (owner is a proc, not a module): emitted via their
## enclosing routine's lambda-lifting, never standalone;
## - generic instances (`sfFromGeneric`): emitted by demand, deduped by merge;
## - `importc`/`compileTime`/`error`/forward sentinels and meta signatures:
## not real codegen targets.
## - method DISPATCHERS (`sfDispatcher`): their bodies are (re)synthesized into
## the main TU by `emitMethodDispatchers`/`generateIfMethodDispatchers`, never
## per module. A dispatcher is a `copySym` clone of the method that shares the
## method's body sub-tree (incl. its closure iterator); transforming it here
## would lambda-lift that SHARED iterator a SECOND time under a different owner
## identity, baking a conflicting `up` field → "up references do not agree"
## (the divergence is impossible in non-IC, where the dispatcher body is empty
## at lift time). So a dispatcher is never an owned runtime routine.
## A `{.closure.}` iterator IS a standalone runtime routine (unlike an inline
## iterator, which is expanded at each call site) and must be emitted by its
## owner — else a cross-module `for` over it links to nothing.
s.itemId.module == modPos and
(s.kind in {skProc, skFunc, skConverter, skMethod} or
(s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and
s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
s.magic == mNone and
sfFromGeneric notin s.flags and
sfDispatcher notin s.flags and
{sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
s.typ != nil and not signatureHasMetaType(s.typ) and
s.ast != nil and s.ast.safeLen > bodyPos and
s.ast[genericParamsPos].kind == nkEmpty and
s.ast[bodyPos].kind != nkEmpty
proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
## Generate C code for a single module.
let moduleId = precomp.module.position
@@ -170,34 +216,7 @@ proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
if g.config.cmd == cmdNifC and g.config.icBackendStage == "cg":
let modPos = precomp.module.position
for s in moduleSymbolStubs(ast.program, FileIndex modPos):
if s.itemId.module == modPos and
s.kind in {skProc, skFunc, skConverter, skMethod} and
# Only MODULE-level routines: a nested/closure proc (its owner is a
# proc) captures its enclosing scope and cannot be emitted standalone —
# the captured params have no loc → `expr: param not init`. Nested procs
# are emitted via their enclosing routine's lambda-lifting, so seeding
# the enclosing (module-level) routine already covers them.
s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
s.magic == mNone and
# Skip generic instances: they have no single owning-module top-level
# and are emitted by demand (emit-everywhere, deduped by the merge
# stage). An instance has an empty `genericParamsPos` just like a plain
# concrete proc, so only `sfFromGeneric` tells them apart; seeding one
# would force standalone codegen of an instance body whose `when T is X`
# branches were never folded for this path → `genMagicExpr: mIs`.
sfFromGeneric notin s.flags and
# Every other routine the module owns must be emitted here, exported or
# not: a non-exported helper is still reached from another module when a
# `template`/inline routine expands at a call site there (e.g. msgs'
# `internalErrorImpl` behind the `internalError` template), and that
# caller now only prototypes it. `{.error.}`/`compileTime` sentinels and
# bodyless forward decls are not real codegen targets.
{sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
s.typ != nil and not signatureHasMetaType(s.typ) and
s.ast != nil and s.ast.safeLen > bodyPos and
s.ast[genericParamsPos].kind == nkEmpty and
s.ast[bodyPos].kind != nkEmpty:
# a concrete, non-generic, runtime routine with a real body, owned here
if ownsRuntimeRoutine(s, modPos):
requestProcDef(bmod, s)
proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
@@ -220,6 +239,16 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
g.config.m.systemFileIdx = systemFileIdx
var precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface})
g.systemModule = precompSys.module
if precompSys.module != nil:
# The precompiled-load path does not restore `sfSystemModule` (mirror of the
# `sfMainModule` re-add above). `registerReusedModuleToMain` keys on it to put
# the system module's init right after its datInit AND to emit
# `initStackBottomWith` into `mainDatInit` — so that the main thread's stack
# bottom is set before any module's init runs. Without the flag the system
# init is mis-routed into the regular `otherModsInit` bucket and
# `initStackBottomWith` is never registered, so a GC cycle during a module's
# init (under refc) scans the stack with a nil bottom and crashes.
incl precompSys.module.flagsImpl, sfSystemModule
var nifFiles: seq[string] = @[toNifFilename(g.config, systemFileIdx)]
var modules = loadModuleDependencies(g, mainFileIdx, nifFiles, depFlags = {})
# loadModuleDependencies traverses the project's import closure and stops at
@@ -323,6 +352,170 @@ proc findTargetModule(g: ModuleGraph; modules: seq[PrecompiledModule];
cachedModuleSuffix(g.config, FileIndex precompSys.module.position) == suffix:
return precompSys
proc setNestedClosureBodies(g: ModuleGraph; idgen: IdGenerator; n: PNode;
owner: PSym; seen: var IntSet) =
## A closure routine nested in `owner` (the `:anonymous` proc lambda-lifting
## minted, plus any deeper nesting) gets its captured-var→env rewrite produced
## as part of the OWNER's `transformBody`. The nested proc is a module-indexed
## sym whose `.s.nif` sdef carries its PRE-lift body, so without help the whole
## module re-serializer would write that pre-lift body and cg would lose the
## capture mapping (it accesses `x` directly instead of `ClE_0->x0`). Walk the
## owner's transformed body and cache each nested closure's transformed body on
## its sym so `writeSymDef` serializes the lifted body into the routine's
## 2-way-body slot.
if n == nil: return
if n.kind == nkSym:
let s = n.sym
if s != nil and s.kind in routineKinds and s != owner and
not seen.containsOrIncl(s.id):
if s.ast != nil and getBody(g, s).kind != nkEmpty and
s.typ != nil and s.typ.callConv == ccClosure:
if s.transformedBody == nil:
s.transformedBody = transformBody(g, idgen, s, {})
setNestedClosureBodies(g, idgen, s.transformedBody, s, seen)
else:
for i in 0 ..< n.safeLen:
setNestedClosureBodies(g, idgen, n[i], owner, seen)
proc reownFromTwin(n: PNode; twin, s: PSym) =
## Re-own to `s` every entity the frontend attributed to `s`'s forward-decl
## `twin` (found via the result's owner). lambda-lifting compares owners by
## reference, so a twin-owned `result` is rejected as `illegalCapture`
## ("'result' ... cannot be captured") and, once that is fixed, twin-owned
## locals go missing from `s`'s env ("environment misses: ..."). Both are
## pervasive on chronos `{.async.}` methods. Re-owning to `s` matches the
## single-sym non-IC case. `twin` is ONE specific sym, so only THIS routine's
## result-twin-owned entities match — re-owning entities of OTHER same-name
## twins proved too blunt (it disrupts env construction and reintroduces the
## very capture errors it should fix). `n.sym != s` guards self-ownership.
if n == nil: return
if n.kind == nkSym and n.sym != nil and n.sym != s and n.sym.owner == twin:
setOwner(n.sym, s)
for i in 0 ..< n.safeLen:
reownFromTwin(n[i], twin, s)
proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend lowering (`--icBackendStage:lower --icBackendModule:<suffix>`):
## enumerate the routines this module OWNS and write them to `<module>.t.nif`.
## Eventually this transforms each owned routine once, in the owner's id space,
## so `cg` reads the result instead of re-deriving it (re-derivation per
## parallel `cg` process is the root of the closure-`:env` identity drift).
## Runs per module in parallel on the shallow backend dep-graph — NOT folded
## into the dense, mostly-serial sem stage.
##
## gate `newSymNode`'s lazy-type marking to the backend (see astdef) — the
## transform builds sym nodes off not-yet-typed stubs, exactly as the `cg`
## stage does.
nifcBackendActive = true
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
else:
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module lowering: module not found for suffix: " & g.config.icBackendModule)
return
let modPos = target.module.position
let tb = BModuleList(g.backend).mods[modPos]
if tb == nil:
rawMessage(g.config, errGenerated,
"per-module lowering: no backend module for suffix: " & g.config.icBackendModule)
return
# Transform every owned routine ONCE in this single process's id space and
# re-serialize the ENTIRE module as a proper indexed NIF (`writeLoweredModule`)
# with the transformed bodies baked into the routine `(sd)` entries. `cg` loads
# it through the normal module loader, so nested procs (incl. async state
# machines) arrive as real defs with their lifted bodies — no re-derivation.
# This single-writer-per-owner is what keeps closure-`:env` identity stable
# across the parallel `cg` processes (re-derivation per process was the root of
# the `:env` identity drift). `transformBody` with flags {} mirrors the cg call
# (cgen.nim); `injectDestructorCalls` is NOT run here — it stays in `cg` on the
# loaded body.
#
# `transformBody`/lambda-lifting LIFTS the closure env's type-bound ops
# (`=destroy` etc.) into `g.opsLog`; snapshot its length so we serialize exactly
# the ops THIS stage created (not those loaded from `.s.nif`).
let opsLogStart = g.opsLog.len
for s in moduleSymbolStubs(ast.program, FileIndex modPos):
if ownsRuntimeRoutine(s, modPos):
# `.s.nif` wins: a routine already transformed during sem (CT eval / macro /
# VM transform) carries its lowered body in the `.s.nif` slot — don't
# re-transform it here.
if s.transformedBody != nil: continue
# A routine serialized as a forward-decl + impl pair (writeSymDef's
# "separate forward declaration and implementation") loads as TWO syms; the
# impl `s` we transform here can carry body entities (`result`, locals,
# nested routines) owned by its fwd-decl TWIN, not by `s`. lambda-lifting
# compares owners by reference → `illegalCapture` rejects a twin-owned
# `result` and the lifting pass can't find twin-owned locals in `s`'s env.
# Pervasive on chronos `{.async.}` methods. Re-own them to `s`, matching the
# single-sym non-IC case. Backend-only, so frontend effect/exception
# inference is untouched.
if s.ast != nil and s.ast.len > resultPos and
s.ast[resultPos].kind == nkSym and s.ast[resultPos].sym.owner != s:
reownFromTwin(s.ast, s.ast[resultPos].sym.owner, s)
# Retain the transformed body on the sym so `writeSymDef` serializes it in
# the routine's `(sd)` 2-way-body slot.
s.transformedBody = transformBody(g, tb.idgen, s, {})
# Cache the lifted body on nested ccClosure routines too, so a module-indexed
# nested closure serializes its lifted (capture-rewritten) body.
var seenNested = initIntSet()
setNestedClosureBodies(g, tb.idgen, s.transformedBody, s, seenNested)
# Collect the hooks this stage lifted, and transform each hook ROUTINE's body
# too (it is itself lowered into NIFC). The hooks' `(sd)` + transformed body go
# into the `.t.nif`; `cg` re-attaches them so `injectDestructorCalls` resolves
# the loaded env's `=destroy`. Iterate to a fixpoint: a hook body can lift
# further hooks (a field's `=destroy`).
var hooks: seq[LogEntry] = @[]
var i = opsLogStart
while i < g.opsLog.len:
let e = g.opsLog[i]
if e.kind == HookEntry and e.sym != nil and e.sym.kind in routineKinds and
e.sym.transformedBody == nil:
hooks.add e
# Transform the hook routine's body and cache it on the sym so `writeSymDef`
# serializes it in the hook's `(sd)` transformed-body slot (`transformBody
# {}` returns the body but does not cache it).
e.sym.transformedBody = transformBody(g, tb.idgen, e.sym, {})
inc i
# Re-serialize the whole module to its suffix-based `.t.nif` (the path
# `toNifFilename` resolves for the cg/emit stages). `writeLoweredModule` seals
# routines itself.
let suffix = cachedModuleSuffix(g.config, FileIndex modPos)
let wholeArtifact = toGeneratedFile(g.config, AbsoluteFile(suffix), ".t.nif").string
writeLoweredModule(ast.program, g.config, target, hooks, wholeArtifact)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icLower] " & extractFilename(wholeArtifact) & " " &
$hooks.len & " hooks"
proc visitDep(suffix: string;
suffixToMod: Table[string, PrecompiledModule];
visited: var HashSet[string]; bl: BModuleList;
ordered: var seq[BModule]) =
## Post-order DFS over a module's import closure used to reconstruct the
## dependency (init) order: a dependency's init must be registered before its
## importer's. Appends each reachable non-main module's `BModule` to `ordered`.
if visited.containsOrIncl(suffix): return
let pm = suffixToMod.getOrDefault(suffix)
if pm.module == nil: return
for dep in pm.deps: # dependencies first (post-order)
visitDep(dep.string, suffixToMod, visited, bl, ordered)
if sfMainModule notin pm.module.flags:
let bm = bl.mods[pm.module.position]
if bm != nil: ordered.add bm
proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend codegen (`--icBackendStage:cg --icBackendModule:<suffix>`):
## generate C for the single module named by `icBackendModule` and write only
@@ -336,6 +529,8 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## module still loads everything (`loadBackendModules`) because NimMain's init
## list and the method dispatchers are whole-program; its `cg` runs essentially
## alone (every other `.c.nif` precedes it), so it does not contend for memory.
# gate `newSymNode`'s lazy-type marking to this stage only (see astdef)
nifcBackendActive = true
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
@@ -363,6 +558,10 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
"per-module codegen: module not found for suffix: " & g.config.icBackendModule)
return
# The `lower` stage already wrote each module's transformed bodies + lifted
# hooks into its `.t.nif`, which the loaders above read directly (toNifFilename
# resolves the `.t.nif`); transformed bodies arrive via loadSymFromCursor and
# lifted hooks via moduleFromNifFile's registerLoadedHooks. Nothing to apply.
generateCodeForModule(g, target)
let bl = BModuleList(g.backend)
# The main module also owns the whole-program method dispatchers + NimMain.
@@ -373,10 +572,58 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# `cg` processes, so the calls are registered here from each `.c.nif` meta
# head — which is why the main module's `cg` runs last, after every other
# `.c.nif` exists. Modules without init code (no `.c.nif`) register nothing.
#
# The registration order IS the runtime init order, and it must be the
# DEPENDENCY (post-order) order: an imported module's init has to run before
# its importer's. The whole-program backend gets this for free — it iterates
# `modulesClosed`, built in module-FINISH order (a post-order DFS over
# imports). Iterating `bl.mods` by position is WRONG: an importer gets a
# LOWER position than the modules it imports (its file is registered before
# its `import` statements are processed), so position order runs importers
# before their dependencies. That left chronicles' `topics_registry` — whose
# init sets `mainThreadId` — running AFTER a module that calls `registerTopic`
# from its own init, tripping the `getThreadId() == mainThreadId` assert at
# startup. So reconstruct the post-order DFS over the import closure here.
#
# NOTE: this is deliberately a SEPARATE traversal rather than reusing the
# module LOAD order — the per-module backend's C emit is sensitive to load
# order (it determines the main TU's header composition), so the loader must
# keep its existing order and the init order is derived independently here.
var suffixToMod = initTable[string, PrecompiledModule]()
for pm in modules:
if pm.module != nil:
suffixToMod[cachedModuleSuffix(g.config, FileIndex pm.module.position)] = pm
if precompSys.module != nil:
suffixToMod[cachedModuleSuffix(g.config, FileIndex precompSys.module.position)] = precompSys
var visited = initHashSet[string]()
var ordered: seq[BModule] = @[]
# System (and its include/import closure) must initialize FIRST: its init
# runs `initGC()` (top-level code in `threadimpl`, included into system),
# and every other module's init may allocate — an allocation before the GC
# heap is set up triggers a collection over an uninitialized region and
# crashes (e.g. nim-metrics' `newRegistry` in its init). System is the
# IMPLICIT universal import and appears in no module's explicit `deps`, so a
# DFS rooted at main never reaches it; seed the traversal from system first.
if precompSys.module != nil:
visitDep(cachedModuleSuffix(g.config, FileIndex precompSys.module.position),
suffixToMod, visited, bl, ordered)
# Then order the whole import closure rooted at the main module; main itself
# is excluded above (its init body becomes NimMain).
for pm in modules:
if pm.module != nil and sfMainModule in pm.module.flags:
visitDep(cachedModuleSuffix(g.config, FileIndex pm.module.position),
suffixToMod, visited, bl, ordered)
# Defensive: any loaded module not reachable from main's import closure
# (demand-loaded system internals) keeps its init registered, appended last
# — nothing imports it, so its relative order does not matter.
for m in bl.mods:
if m != nil and sfMainModule notin m.module.flags:
let heads = readCnifHeads(getCFile(m).string & ".nif")
registerReusedModuleToMain(bl, m, heads.initRequired, heads.datInitRequired)
let suffix = cachedModuleSuffix(g.config, FileIndex m.module.position)
if not visited.containsOrIncl(suffix):
ordered.add m
for m in ordered:
let heads = readCnifHeads(getCFile(m).string & ".nif")
registerReusedModuleToMain(bl, m, heads.initRequired, heads.datInitRequired)
let tb = bl.mods[target.module.position]
if tb != nil:
finishModule(g, tb)
@@ -457,7 +704,17 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
let artifact = cfile & ".nif"
var dropped = 0
let code = renderCFromArtifact(artifact, decision, extractFilename(artifact), dropped)
writeFile(cfile, code)
# Write the `.c` content-stably. `merge` re-runs on any edit and bumps the
# decision file's mtime, so nifmake re-fires every `emit` (the filter is cheap);
# but the FILTERED output is usually byte-identical for modules unaffected by
# the edit. Rewriting it unconditionally would bump every `.c`'s mtime and make
# `callCCompiler` recompile every `.o`. Writing only on a real change preserves
# the mtime, so the C compiler recompiles exactly the modules whose `.c` changed
# — the same DCE model as Nimony's. Safe here (unlike a content-stable merge
# decision): a `.c` is a per-module LEAF consumed only by the C compiler's own
# up-to-date check, not a shared prerequisite in nifmake's mtime ordering.
if not fileExists(cfile) or readFile(cfile) != code:
writeFile(cfile, code)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icEmit] " & extractFilename(cfile) & " dropped " &
$dropped & " bodies (" & $code.len & " bytes)"
@@ -483,6 +740,7 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
if precompSys.module != nil:
replayBackendActions(g, precompSys.module, precompSys.topLevel)
let bl = BModuleList(g.backend)
var addedCFiles = initHashSet[string]()
for m in bl.mods:
if m != nil:
let cfile = getCFile(m)
@@ -490,17 +748,53 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# (extra members of system's closure that no build rule targets) had their
# code emit-everywhere'd into the targets, so they have no file to compile.
if not fileExists(cfile.string): continue
addedCFiles.incl extractFilename(cfile.string)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
addFileToCompile(g.config, cf)
# `addExternalFileToCompile` (not `addFileToCompile`) gates each `.c` on its
# SHA1 footprint: an unchanged `.c` keeps its `.o` and is flagged Cached, so
# `callCCompiler` skips its compile but still links the existing object. This
# is what makes a localized edit recompile only the handful of `.c`s the
# `emit` stage actually rewrote, instead of every object every time — the
# final piece of per-module backend incrementality after the merge barrier.
addExternalFileToCompile(g.config, cf)
# deps.nim's static scanner can keep a CONDITIONALLY-imported module as a build
# node (e.g. `net`'s `when defineSsl: import openssl`, or a `when defined(os)`
# import) that the NIF-`deps` walk above never reaches because the condition is
# off. Such a node still emitted a `.c`, and it can OWN a live generic instance
# that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused by
# `strutils.escape`) — so its body must be at link or that reference is
# undefined. Link every emitted `.c` the merge decision says OWNS a LIVE symbol;
# a node that owns nothing live (a Windows-only winsock node on Linux) is
# correctly skipped.
block:
let nimcache = getNimcacheDir(g.config).string
let decision = readMergeDecision(nimcache / MergeDecisionFile)
if not decision.broken:
var liveOwners = initHashSet[string]()
for cname, owner in decision.owners:
if owner.endsWith(".c.nif") and cname in decision.live:
liveOwners.incl owner
for owner in liveOwners:
let cbase = owner[0 ..< owner.len - ".nif".len] # "@m….nim.c.nif" -> ".c"
if addedCFiles.containsOrIncl(cbase): continue
let cfile = AbsoluteFile(nimcache / cbase)
if not fileExists(cfile.string): continue
var cf = Cfile(nimname: cbase, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
addExternalFileToCompile(g.config, cf)
if g.config.cmd != cmdTcc:
extccomp.callCCompiler(g.config)
proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
## Main entry point for NIF-based C code generation.
## Traverses the module dependency graph and generates C code.
if g.config.icBackendStage == "cg":
if g.config.icBackendStage == "lower":
generateLowerStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "cg":
generateCgStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "merge":
@@ -514,4 +808,4 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
return
else:
rawMessage(g.config, errGenerated,
"the per-module NIF backend requires --icBackendStage:cg|merge|emit|link")
"the per-module NIF backend requires --icBackendStage:lower|cg|merge|emit|link")

2
compiler/options.nim
View File

@@ -29,7 +29,7 @@ const
nimEnableCovariance* = defined(nimEnableCovariance)
icFormatVersion* = "6"
icFormatVersion* = "21"
## Version of the IC cache format (the sem-NIF module layout written by
## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
## that layout changes: `commandIc` wipes a nimcache whose `ic.version`

75
compiler/pipelines.nim
View File

@@ -298,6 +298,16 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
if not hasNil:
genericOffers.add (inst.sym.instantiatedFrom, inst.sym,
inst.concreteTypes, inst.genericParamsCount)
# Generic TYPE-instance OFFERS: every `tyGenericInst` THIS module created,
# so a consumer reuses its baked structure (array bounds etc.) rather than
# re-instantiating with a scope-divergent bound. See ast2nif.writeNifModule.
var typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[]
for genItemId, instList in graph.typeInstCache:
for inst in instList:
if inst != nil and inst.uniqueId.module == module.position and
inst.kidsLen > 0 and inst[0] != nil and
inst[0].kind == tyGenericBody and inst[0].sym != nil:
typeOffers.add (inst[0].sym, inst)
# The module's REAL resolved direct imports (incl. macro/template-generated
# ones with no surviving syntactic node). Passed to writeNifModule so the
# NIF `deps` section is complete (the backend closure walk needs it), and
@@ -305,7 +315,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
let resolvedImportDeps = graph.importDeps.getOrDefault(module.position.FileIndex, @[])
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps, reexportedModuleSyms(graph, module),
genericOffers, resolvedImportDeps)
genericOffers, typeOffers, resolvedImportDeps)
# The module's REAL direct imports (incl. macro-generated) for `nim ic`'s
# graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep.
var semDepPaths: seq[string] = @[]
@@ -351,6 +361,31 @@ proc initLoadedCompileTimeGlobals(graph: ModuleGraph; module: PSym; topLevel: PN
sect.add s.ast
setupCompileTimeVar(module, idgen, graph, sect)
proc finalizeLoadedModules(graph: ModuleGraph) =
## Apply the VM-level load effects of every module just loaded from a NIF —
## direct import OR dep-of-a-dep, both collected in `graph.pendingNifInit` by the
## loader (modulegraphs.moduleFromNifFile / loadTransitiveHooks). This is the ONE
## place that knows what loading a module does to global VM state, so a
## transitively-reached module (which never passes through this proc's caller)
## gets identical treatment. Modules are in dependency order (deps before
## dependents), which is the correct macro-cache replay order.
## 1. macro-cache replay: std/macrocache put/inc/add/incl recorded in the
## module's top level (pragma replay actions are a backend concern, skipped).
## 2. eager `{.compileTime.}` global init (see initLoadedCompileTimeGlobals).
## To add a new per-load effect, extend this proc — do not add a parallel buffer.
if graph.pendingNifInit.len == 0: return
for (m, topLevel) in graph.pendingNifInit:
if topLevel == nil: continue
var replayList = newNodeI(nkStmtList, m.info)
for n in topLevel:
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
replayList.add n
if replayList.len > 0:
replayStateChanges(m, graph, replayList)
initLoadedCompileTimeGlobals(graph, m, topLevel)
graph.pendingNifInit.setLen 0
proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags; fromModule: PSym = nil): PSym =
var flags = flags
if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
@@ -420,33 +455,12 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
if sfSystemModule in flags:
graph.systemModule = result
partialInitModule(result, graph, fileIdx, AbsoluteFile(toFullPath(graph.config, fileIdx)))
# Replay the module's recorded state changes: macro-cache operations
# (std/macrocache puts/incs/adds/incls) plus a few pragmas. The loader
# parsed them into `precomp.topLevel` (mixed with other top-level nodes),
# so filter to the replay actions. A loaded module's `ast` is never
# rebuilt, so this used to be skipped (`result.ast == nil`) and a
# NIF-loaded module's macro cache was lost — e.g. nim-serialization's
# flavor registration became invisible to dependents (`DefaultFlavor:
# automatic serialization is not enabled`).
var replayList = newNodeI(nkStmtList, result.info)
for n in precomp.topLevel:
# Only macro-cache ops (put/inc/add/incl). The pragma replay actions
# (compile/link/passc/hint/...) are a backend/link concern handled by
# the nifc closure, and re-emitting a loaded module's hints/warnings on
# every import would be wrong — so they are deliberately skipped here.
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
replayList.add n
# Plus the macro-cache actions of the module's transitive import closure
# (collected by the moduleFromNifFile call above via loadTransitiveHooks),
# so a flavor/type registered in an indirectly-imported module is visible.
for n in graph.transitiveReplayActions: replayList.add n
graph.transitiveReplayActions.setLen 0
if replayList.len > 0:
replayStateChanges(result, graph, replayList)
# Fill the VM slots of the module's `{.compileTime.}` globals now (sem
# would have, but a NIF-loaded module is never semchecked).
initLoadedCompileTimeGlobals(graph, result, precomp.topLevel)
# Apply the VM-level load effects of this module AND every dep it pulled in
# (moduleFromNifFile recorded them all in graph.pendingNifInit): macro-cache
# replay (else a NIF-loaded module's macro cache is lost — e.g.
# nim-serialization flavor registration) and eager `{.compileTime.}` global
# init. Uniform for direct and transitive deps — see finalizeLoadedModules.
finalizeLoadedModules(graph)
return result # Return early, don't process from source
let path = toFullPath(graph.config, fileIdx)
let filename = AbsoluteFile path
@@ -551,6 +565,11 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
localError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for system module (expected: " & nifPath & ")")
return
# Apply system's (and its deps') load effects now: the main module is
# compiled from source and never re-enters the moduleFromNifFile drain for
# system, so without this its macro-cache / CT globals would wait until the
# first NIF import is processed. See finalizeLoadedModules.
finalizeLoadedModules(graph)
discard graph.compilePipelineModule(projectFile, {sfMainModule})
else:
graph.compilePipelineSystemModule()

39
compiler/sempass2.nim
View File

@@ -1117,6 +1117,45 @@ proc trackCall(tracked: PEffects; n: PNode) =
#if canRaise(a):
# echo "this can raise ", tracked.config $ n.info
let op = a.typ
# A routine whose body reaches a compile-time-only magic (`macros.error`,
# `slurp`, `gorge`, `getAst`, …) can never be code-generated — the C/JS
# backends reject those magics (ccgexprs `errXMustBeCompileTime`). Such a
# routine is compile-time-only by construction; mark it `sfCompileTime` so it
# is treated uniformly as such. Non-IC pruned it by demand-driven codegen, but
# the per-module IC backend emits every owned routine (no DCE) and would
# otherwise feed the magic to codegen. Mirrors the `tfTriggersCompileTime ->
# sfCompileTime` path in `semProcAux`.
if a.kind == nkSym and a.sym.magic in {mNLen..mNError, mSlurp..mQuoteAst} and
tracked.owner != nil and tracked.owner.kind in routineKinds and
tracked.config.cmd != cmdNimscript:
# ...but NOT under `nim e`: nimscript has no codegen backend to protect, and
# marking a routine `sfCompileTime` makes `semExpr` eagerly fold calls to it
# at sem time (emConst), where module-level globals it reads have no VM slot
# yet — distros' `detectOsWithAllCmd` reaches `gorge` and reads the plain
# global `unameRes` → "cannot evaluate at compile time: unameRes". In the
# normal nimscript run (emRepl) the module's var section runs first and the
# slot exists, so the marking is both unnecessary and harmful here.
#
# ...and NOT if the routine is — or is nested inside — a macro/template:
# those are VM-only (never code-generated), so the per-module IC backend has
# nothing to protect there, while `sfCompileTime` on a macro-internal nested
# closure breaks its captured-variable access in the VM ("cannot evaluate at
# compile time: n" — `tests/macros/tmacros1`'s `innerProc` reading the
# macro-local `n`). Walk the owner chain and bail on the first
# skMacro/skTemplate. NB mark `tracked.owner` (the routine that directly
# reaches the magic), NOT its outermost enclosing: a runtime proc may legally
# nest a compile-time helper — `tests/generics/tunique_type`'s `[]` proc
# contains a nested `buildResult` macro — and marking the proc would wrongly
# make IT compile-time ("request to generate code for .compileTime proc: []").
var encl = tracked.owner
var insideMeta = false
while encl != nil and encl.kind != skModule:
if encl.kind in {skMacro, skTemplate}:
insideMeta = true
break
encl = encl.skipGenericOwner
if not insideMeta:
incl(tracked.owner, sfCompileTime)
if n.typ != nil:
if tracked.owner.kind != skMacro and n.typ.skipTypes(abstractVar).kind != tyOpenArray:
createTypeBoundOps(tracked, n.typ, n.info)

24
compiler/semstmts.nim
View File

@@ -2621,6 +2621,17 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
addParams(c, proto.typ.n, proto.kind)
proto.info = s.info # more accurate line information
proto.options = s.options
# `s` (the impl symbol) is discarded in favour of `proto`. It still carries
# `s.ast == n` (set above) and stays reachable as the owner of body-local
# symbols, so under IC it would be serialized as a SECOND, body-bearing
# `proc` entry — a phantom duplicate of `proto`. The per-module backend then
# codegens that phantom, whose `result` is owned by `proto` (addResult below
# re-parents it), not by the phantom: lambdalifting's capture check
# (`result.skipGenericOwner != owner`) then wrongly classifies `result` as a
# captured outer variable → "'result' … cannot be captured". Drop the
# discarded impl's body so it can never be emitted as a routine (same leak
# class the `miscPos` adoption below guards against for generic params).
let discardedImpl = s
s = proto
n[genericParamsPos] = proto.ast[genericParamsPos]
n[paramsPos] = proto.ast[paramsPos]
@@ -2638,6 +2649,19 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
if importantComments(c.config) and proto.ast.comment.len > 0:
n.comment = proto.ast.comment
proto.ast = n # needed for code generation
if discardedImpl != proto:
discardedImpl.ast = nil
# The impl symbol is discarded in favour of `proto`, but it stays `Complete`
# in this module, so `ast2nif.shouldWriteSymDef` still serializes it. With
# `sfExported` it would be written importable (`x` marker) and an importer
# would load BOTH it and `proto` into the overload set: "ambiguous call;
# both foo and foo" (identical signatures). Normally a discarded impl is a
# gensym/transient that isn't reached this way, but a `{.async: (raises).}`
# forward-decl + impl reconciles HERE with both syms exported. Strip the
# export so the design's "forward declarations are never importable" holds —
# the def still serializes (other refs may resolve to it) but is invisible
# to importer overload resolution; `proto` carries the export.
excl(discardedImpl, sfExported)
popOwner(c)
pushOwner(c, s)

26
compiler/semtypinst.nim
View File

@@ -480,7 +480,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
else:
header = instCopyType(cl, t)
result = newType(tyGenericInst, cl.c.idgen, t.genericHead.owner, son = header.genericHead)
# The instantiating module owns the instance (and announces it as an offer):
# the generic body's module (`t.genericHead.owner`) has no business owning a
# type that references instantiation-site types — that is the IC parent->child
# heap leak the write-barrier surfaces.
result = newType(tyGenericInst, cl.c.idgen, cl.c.module, son = header.genericHead)
result.flags = header.flags
# be careful not to propagate unnecessary flags here (don't use rawAddSon)
# ugh need another pass for deeply recursive generic types (e.g. PActor)
@@ -834,15 +838,21 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType, isInstValue = false):
# trough replaceObjBranches in order to resolve any pending nkRecWhen nodes
result = t
# Slow path, we have some work to do
if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and t.elementType.n != nil:
# Slow path, we have some work to do. CRUCIAL: only ever mutate a type that
# is LOCAL to the module we are instantiating in (`uniqueId.module ==
# idgen.module`). A type loaded from another module's NIF (foreign) already
# had its object branches resolved when it was originally compiled; mutating
# it in place here is an old→new heap write that re-homes the loaded type to
# the instantiation site (its sym then looks owned by the consumer module and
# loses its `info`, colliding C type names — the libp2p `Message` bug). The
# prior `state != Sealed` guard was insufficient: a freshly-LOADED type is
# `Complete`, not `Sealed` (`Sealed` only means "already re-written to a NIF").
if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and
t.elementType.n != nil and t.elementType.uniqueId.module == cl.c.idgen.module.int:
discard replaceObjBranches(cl, t.elementType.n)
elif result.n != nil and t.kind == tyObject and result.state != Sealed:
# A type loaded from the IC cache already had its object branches
# resolved when it was originally compiled, and must not be mutated in
# place (nor copied, which would break object-inheritance identity), so
# only non-Sealed types are processed here.
elif result.n != nil and t.kind == tyObject and result.state != Sealed and
result.uniqueId.module == cl.c.idgen.module.int:
# Invalidate the type size as we may alter its structure
result.size = -1
result.n = replaceObjBranches(cl, result.n)

15
compiler/sighashes.nim
View File

@@ -10,6 +10,7 @@
## Computes hash values for routine (proc, method etc) signatures.
import ast, ropes, modulegraphs, options, msgs, pathutils
from lineinfos import FileIndex
from std/hashes import Hash
import std/tables
import types
@@ -74,7 +75,19 @@ proc hashTypeSym(c: var MD5Context, s: PSym; conf: ConfigRef) =
c &= ":anon"
else:
var it = s
c &= customPath(conf.toFullPath(s.info))
# The source file path disambiguates same-named object types from different
# modules whose owner-chain names also coincide (e.g. libp2p kademlia/protobuf
# `Message` vs rendezvous/protobuf `Message`, both modules named `protobuf`).
# A type sym that reaches the backend as a `Complete` stub never individually
# loaded carries `unknownLineInfo` (fileIndex -1), which `toFullPath` collapses
# to the `???` placeholder — so the two would hash to ONE mangled C name and the
# wrong struct gets emitted. Fall back to the sym's HOME module file (its
# per-module NIF-suffix path, stable+unique) for the path. Only fires on a -1
# fileIndex; non-IC type syms always have a real `info`, so the fast path is
# taken and the hash is unchanged (koch boot byte-equal).
let infoFi = s.info.fileIndex
let pathFi = if infoFi.int32 >= 0'i32: infoFi else: s.itemId.module.int32.FileIndex
c &= customPath(conf.toFullPath(pathFi))
when defined(icDbgHash):
var ownerSteps = 0
while it != nil:

30
compiler/transf.nim
View File

@@ -1386,7 +1386,33 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flags: Transf
result = getBody(g, prc)
else:
prc.transformedBody = newNode(nkEmpty) # protects from recursion
var c = openTransf(g, prc.getModule, "", idgen, flags)
# Lambda-lifting a routine body while the VM compiles it (to run a macro
# under `nim ic`) mints a closure `:env` (type + obj + fields + hidden param)
# that the lift welds into the routine's serialized signature. Such an env is
# a PROCESS-LOCAL artifact (its item number is per-process-sequential), so a
# reference to it must never carry a stable cross-module identity — otherwise
# a consumer resolves it against a canonical NIF built by a different process
# that has no matching def ('symbol has no offset', e.g. Nimbus t17.275).
# Lift in the backend (process-local) id space; ast2nif then emits these as
# module-local `@bk` defs (mirrors setAttachedOp's inVMTransform handling).
var liftIdgen = idgen
if g.inVMTransform > 0 and g.config.cmd == cmdM:
if g.vmTransfIdgen == nil:
g.vmTransfIdgen = idGeneratorForBackend(g.systemModule)
liftIdgen = g.vmTransfIdgen
var c = openTransf(g, prc.getModule, "", liftIdgen, flags)
# `liftCapturedVars` rewrites captured locals to `:env.field` IN PLACE on the
# body it is handed; the env-creation prologue lands only in the returned
# wrapper. When the VM drives this transform (running a macro/CT proc), that
# in-place mutation corrupts the routine's PRE-transform `ast[bodyPos]` —
# under IC exactly the node `getBody` serializes to the module's `.s.nif`. So
# snapshot the pristine body before the VM lift and restore `ast[bodyPos]`
# afterwards: the VM still consumes the fully-lifted `result`, but `getBody`
# keeps faithfully returning the pre-transform body for serialization. The
# cg/backend path (`inVMTransform == 0`) is untouched.
let vmPristineBody =
if g.inVMTransform > 0: copyTree(getBody(g, prc))
else: nil
result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen, flags)
result = processTransf(c, result, prc)
liftDefer(c, result)
@@ -1396,6 +1422,8 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flags: Transf
result = g.transformClosureIterator(c.idgen, prc, result)
incl(result.flags, nfTransf)
if vmPristineBody != nil:
prc.ast[bodyPos] = vmPristineBody
if useCache in flags or prc.typ.callConv == ccInline:
# genProc for inline procs will be called multiple times from different modules,

27
compiler/typekeys.nim
View File

@@ -139,6 +139,25 @@ proc maybeImported(c: var Context; s: PSym; conf: ConfigRef) {.inline.} =
if s != nil and {sfImportc, sfExportc} * s.flagsImpl != {}:
c.symKey(s, conf)
proc backendTypeName(t: PType; conf: ConfigRef): string =
## Stable cross-module identity of a backend-minted (lower-stage) type: its
## serialized `@bk` NIF name (mirrors ast2nif.nifTypeName). A closure-env
## object/ref minted by the `lower` stage has NO stable STRUCTURAL key — its
## captured-field types re-resolve to different modules in the producing vs the
## consuming process (e.g. field `x0` → `int` in the producer, → the consumer's
## alias in the consumer) — but this name (kind + item + home-module suffix) is
## identical in both, because the consumer loads the producer's name verbatim.
## Keying hooks by it makes producer `setAttachedOp` and consumer `getAttachedOp`
## agree. The trailing `@bk` (= ast2nif.BackendLocalMarker) keeps it disjoint
## from any normal type's structural key.
result = "`t"
result.addInt ord(t.kind)
result.add '.'
result.addInt t.uniqueId.item
result.add '.'
result.add modname(t.uniqueId.module, conf)
result.add "@bk"
proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef) =
if t == nil:
c.m.addEmpty()
@@ -148,6 +167,14 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
assert c.tl != nil
c.tl(t)
if t.uniqueId.isBackendMinted:
# Backend-minted (lower-stage) closure-env types key by their stable NIF name,
# never by structure (which diverges across the NIF boundary). An env `ref`
# that is itself NOT backend-minted still keys stably: it recurses here and
# reaches its `@bk` object, which short-circuits to a stable name.
c.m.addSymbol backendTypeName(t, conf)
return
case t.kind
of tyGenericInvocation:
for a in t.sonsImpl:

37
compiler/vm.nim
View File

@@ -1313,8 +1313,41 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
# a macro observed this symbol's implementation: NeedsImpl edge to
# its home module under IC.
recordIcImplDep(c.graph, a.sym)
regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit)
else: copyTree(a.sym.ast)
if a.sym.ast.isNil:
regs[ra].node = newNode(nkNilLit)
else:
let tree = copyTree(a.sym.ast)
# A NIF-loaded routine's `ast[paramsPos]` is an `nkEmpty` placeholder:
# ast2nif strips the formal params (recoverable from `typ.n`, see
# writeNode's `skipParams`). A macro that reads `fn.getImpl[paramsPos]`
# — e.g. taskpools `spawn` reads the return type via `getImpl[3][0]` —
# needs them, so reconstruct a read-only formalParams from the proc
# type. The synthesized type-expression nodes carry the resolved
# `PType`, which is all a macro can query for a loaded routine.
if tree.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
nkConverterDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo} and
tree.safeLen > paramsPos and tree[paramsPos].kind == nkEmpty and
a.sym.typ != nil and a.sym.typ.n != nil and
a.sym.typ.n.kind == nkFormalParams:
let t = a.sym.typ
let fp = newNodeI(nkFormalParams, a.sym.info)
let rt = t.returnType
# `opMapTypeInstToAst` (inst=true) reproduces a source-like type
# declaration — crucially it renders an array's range bound as
# `range 0..N` (the `inst=false` form emits `range[0, N]`, which
# re-sems to "'range' expects one type parameter").
fp.add(if rt != nil: opMapTypeInstToAst(c.cache, rt, a.sym.info, c.idgen)
else: newNodeI(nkEmpty, a.sym.info))
for i in 1 ..< t.n.len:
if t.n[i].kind == nkSym:
let p = t.n[i].sym
let def = newNodeI(nkIdentDefs, p.info)
def.add newIdentNode(p.name, p.info)
def.add opMapTypeInstToAst(c.cache, p.typ, p.info, c.idgen)
def.add newNodeI(nkEmpty, p.info)
fp.add def
tree[paramsPos] = fp
regs[ra].node = tree
regs[ra].node.flags.incl nfIsRef
else:
stackTrace(c, tos, pc, "node is not a symbol")

3
koch.nim
View File

@@ -618,7 +618,8 @@ proc runIcTestFile(inp: string) =
# which exercises the NIF import/load path the single-file tests do not.
const icSuite = ["thallo", "tconverter", "timp", "tmiscs", "tparseutils",
"tcompiletimeglobal", "tsighashstable", "tpureenum", "tgenericoffer",
"tconverterreexport"]
"tconverterreexport", "ttypeoffer", "ttransitiveoffer",
"tmodsymref", "tmethupref"]
proc icTest(args: string) =
temp("")

23
tests/ic/mmethupref.nim Normal file
View File

@@ -0,0 +1,23 @@
# Helper for tmethupref: a `{.base.}` method whose body contains a closure
# iterator that in turn contains a nested closure capturing a method-local.
# The capture forces an `up` reference chain (nested closure -> iterator env ->
# method env). The method also gets a dispatcher (a `copySym` clone that shares
# the method's body sub-tree, including the iterator). Under `nim ic` the
# dispatcher used to be treated as an owned runtime routine and lambda-lifted in
# the per-module lower stage, lifting the SHARED iterator a second time under a
# different owner identity -> "up references do not agree" / "could not determine
# closure type". See nifbackend.ownsRuntimeRoutine (sfDispatcher exclusion).
type Base* = ref object of RootObj
val*: int
method compute*(b: Base): int {.base.} =
var acc = b.val
iterator steps(): int {.closure.} =
proc bump() =
acc += 1
bump()
bump()
yield acc
for s in steps():
result = s

12
tests/ic/mmodsymadd.nim Normal file
View File

@@ -0,0 +1,12 @@
import mmodsymasm
# The dead `else` branch qualifies a re-exported module (`mmodsymarm`) whose
# `foo` is gated out on this host: the module sym binds at template-definition
# but the member never resolves, so a dangling module-symbol reference survives
# into the serialized template body. Before the `ModMarker` fix this failed
# under `nim ic` with `symbol has no offset` when the consumer loaded this NIF.
template satAdd*(a, b: uint64): uint64 =
when not defined(mmodSymFakeArch):
mmodsymasm.mmodsymx86.foo(a, b)
else:
mmodsymasm.mmodsymarm.foo(a, b)

5
tests/ic/mmodsymarm.nim Normal file
View File

@@ -0,0 +1,5 @@
# Gated out on every host (mmodSymFakeArch is never defined): `foo` does NOT
# exist here, so a qualified `…mmodsymarm.foo` cannot resolve to the proc and
# leaves the bare re-exported MODULE symbol dangling in the template body.
when defined(mmodSymFakeArch):
func foo*(a, b: uint64): uint64 = a + b

2
tests/ic/mmodsymasm.nim Normal file
View File

@@ -0,0 +1,2 @@
import mmodsymx86, mmodsymarm
export mmodsymx86, mmodsymarm

3
tests/ic/mmodsymx86.nim Normal file
View File

@@ -0,0 +1,3 @@
# Live on every real host: provides `foo` so the template's taken branch resolves.
when not defined(mmodSymFakeArch):
func foo*(a, b: uint64): uint64 = a + b

3
tests/ic/mtoffcodec.nim Normal file
View File

@@ -0,0 +1,3 @@
# Helper for ttypeoffer.nim: the extra overload that flips `compiles(toSszType(Gwei))`.
import mtoffgwei
template toSszType*(v: Gwei): uint64 = uint64(v)

2
tests/ic/mtoffgwei.nim Normal file
View File

@@ -0,0 +1,2 @@
# Helper for ttypeoffer.nim: a distinct basic type.
type Gwei* = distinct uint64

10
tests/ic/mtoffssz.nim Normal file
View File

@@ -0,0 +1,10 @@
# Helper for ttypeoffer.nim: a generic container whose hash-array bound depends
# on a `mixin toSszType` resolved at the instantiation site (cf. ssz dataPerChunk).
template perChunk*(T: type): int =
mixin toSszType
when compiles(toSszType(default(T))): 4 else: 1
type
HA*[N: static int; T] = object
data*: array[N, T]
hashes*: array[N div perChunk(T), uint64]

5
tests/ic/mtoffstate.nim Normal file
View File

@@ -0,0 +1,5 @@
# Helper for ttypeoffer.nim ("datatypes" analog): instantiates HA[64,Gwei] WITHOUT
# the codec in scope -> perChunk=1. This is the instance that must be shared.
import mtoffssz, mtoffgwei
type StateA* = object
field*: HA[64, Gwei]

10
tests/ic/mtoffuser.nim Normal file
View File

@@ -0,0 +1,10 @@
# Helper for ttypeoffer.nim ("db_immutable" analog): imports the codec (so
# `toSszType(Gwei)` is visible -> perChunk=4) and re-instantiates HA[64,Gwei].
# With the `(toffer …)` fix it reuses mtoffstate's instance instead.
import mtoffssz, mtoffgwei, mtoffcodec, mtoffstate
type StateB* = object
field*: HA[64, Gwei]
proc check*() =
static: doAssert sizeof(StateA) == sizeof(StateB)
echo "ok"

2
tests/ic/mtscopea.nim Normal file
View File

@@ -0,0 +1,2 @@
# Helper for ttransitiveoffer.nim: const that the generic body binds at definition.
const TScopeSize* = 65

2
tests/ic/mtscopeb.nim Normal file
View File

@@ -0,0 +1,2 @@
# Helper: a CONFLICTING const of the same name, visible only in the consumer.
const TScopeSize* = 33

7
tests/ic/mtscopegen.nim Normal file
View File

@@ -0,0 +1,7 @@
# Helper: defines a generic whose body uses TScopeSize via the strformat `&`
# macro (late-bound), resolved in THIS module's scope (mtscopea -> 65).
import mtscopea, std/strformat
export mtscopea
func fromRaw*[T](x: T): string =
const msg = &"size {TScopeSize - 1}"
result = msg & " " & $int(x)

3
tests/ic/mtscopemid.nim Normal file
View File

@@ -0,0 +1,3 @@
# Helper: makes mtscopewarm a TRANSITIVE import of the consumer.
import mtscopewarm
export mtscopewarm

4
tests/ic/mtscopewarm.nim Normal file
View File

@@ -0,0 +1,4 @@
# Helper: instantiates fromRaw[int] in a CLEAN scope (no mtscopeb) -> the
# correct instance that the consumer must reuse.
import mtscopegen
proc warm*(): string = fromRaw(5)

15
tests/ic/tmethupref.nim Normal file
View File

@@ -0,0 +1,15 @@
discard """
output: '''42'''
"""
# Regression test: a `{.base.}` method whose body holds a closure iterator with a
# nested capturing closure must not crash the IC backend. The method's dispatcher
# (a `copySym` clone sharing the iterator) must NOT be lambda-lifted per module;
# otherwise the shared iterator's `up` field is baked twice under divergent owner
# identities -> "up references do not agree" / "could not determine closure type"
# (the real-world symptom: ~all libp2p async `{.base.}` methods failed under
# `nim ic`). Fixed by excluding `sfDispatcher` from nifbackend.ownsRuntimeRoutine.
import mmethupref
let b = Base(val: 40)
echo b.compute()

12
tests/ic/tmodsymref.nim Normal file
View File

@@ -0,0 +1,12 @@
discard """
output: '''7'''
"""
# Regression test: a cross-module MODULE-symbol reference left as a dangling
# qualifier in a template body (here `mmodsymasm.mmodsymarm.foo` in a dead
# `when`-branch, reached via re-export) must load under `nim ic` instead of
# raising `symbol has no offset`. Mirrors nim-intops' `inlineasm.arm64.X` in
# nimbus-eth2. See compiler/ast2nif.nim `ModMarker`.
import mmodsymadd
echo satAdd(3'u64, 4'u64)

14
tests/ic/ttransitiveoffer.nim Normal file
View File

@@ -0,0 +1,14 @@
discard """
output: '''size 64 64'''
"""
# Regression test for TRANSITIVE generic-instance offers. `fromRaw[int]` is first
# instantiated in mtscopewarm (a clean scope where `TScopeSize` is unambiguously
# 65). The consumer here also imports mtscopeb (`TScopeSize` = 33), so a fresh
# re-instantiation of fromRaw's body would resolve `TScopeSize` ambiguously. The
# clean instance reaches here only TRANSITIVELY (via mtscopemid), so the offer
# rebuild must walk the whole import closure, not just direct imports. Mirrors
# nimbus-eth2 `keys.fromRaw` -> `SkRawPublicKeySize` (secp vs secp256k1).
import mtscopegen, mtscopeb, mtscopemid
echo fromRaw(64)

14
tests/ic/ttypeoffer.nim Normal file
View File

@@ -0,0 +1,14 @@
discard """
output: '''ok'''
"""
# Regression test for the `(toffer …)` generic-TYPE-instance sharing across the
# NIF boundary. A `tyGenericInst` whose structure (here an `array` bound) depends
# on a `mixin`/`compiles()` resolved at the instantiation site must be REUSED
# from the module that created it, not re-instantiated in a consumer whose import
# scope flips the `compiles()` and so bakes a different bound. Mirrors the SSZ
# `HashArray[8192, Gwei]` `sizeof` divergence in nimbus-eth2. Before the fix this
# failed under `nim ic` with `doAssert sizeof(StateA) == sizeof(StateB)`.
import mtoffuser
check()