diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index b2525510d5..e95d9b9fbc 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -198,6 +198,10 @@ type depSuffixes: HashSet[string] # module suffixes already emitted as `(import ...)` deps emittedBackendTypes: HashSet[int32] # backend-local type items already def'd this module emittedBackendSyms: HashSet[int32] # backend-local sym items already def'd this module + lowering: bool # serializing the `lower` stage's `.t.nif` (per-entry self-contained) + emittedFieldSyms: HashSet[ItemId] # lowering: derived env-field syms already def'd this entry + inTypeReclist: int # >0 while writing a type's OWN reclist: fields must be SELF-CONTAINED + # defs (the type can be seek-loaded in isolation), not entry-deduped uses proc isLocalSym(sym: PSym): bool {.inline.} = @@ -377,7 +381,13 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = writeType(w, dest, typ.typeInstImpl) #if typ.kind in {tyProc, tyIterator} and typ.nImpl != nil and typ.nImpl.kind != nkFormalParams: + # The reclist holds this type's OWN fields. A type can be force-loaded by + # name in isolation (cg seeks the `.t.nif`/`.s.nif` index entry), so its + # fields must be DEFS here, not entry-deduped SymUses whose def lives + # elsewhere in the `(lowered)` entry and is never read by the seek. + inc w.inTypeReclist writeNode(w, dest, typ.nImpl) + dec w.inTypeReclist writeSym(w, dest, typ.ownerFieldImpl) writeSym(w, dest, typ.symImpl) @@ -497,6 +507,18 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeLoc w, dest, sym.locImpl writeNode(w, dest, sym.constraintImpl) writeSym(w, dest, sym.instantiatedFromImpl) + # The TRANSFORMED body (ic_ideas.md 2-way body): a routine run at compile time + # (macro / VM transform / `static`) already has its lowered body — closure + # `:env` and all — computed during sem; serialize it so the backend reuses it + # instead of re-deriving (the divergence behind the t17.275 env class). An + # empty `.` here means "same as the semchecked body OR to be found in the + # `.t.nif`" (the `lower` stage fills that gap). Non-routines / not-yet- + # transformed routines write the empty marker. (`transformedBodyImpl` only + # exists in the routine branch of the `TSym` variant.) + if sym.kindImpl in routineKinds: + writeNode(w, dest, sym.transformedBodyImpl) + else: + dest.addDotToken dest.addParRi @@ -517,9 +539,24 @@ proc shouldWriteSymDef(w: var Writer; sym: PSym): bool {.inline.} = return true # Normal case for global symbols return false +proc isLoweredPerEntryField(w: Writer; sym: PSym): bool {.inline.} = + ## In the `lower` stage every entry (each `(lowered)` body AND each `@bk` type + ## def, which carries its fields inline) is loaded independently, so it must be + ## SELF-CONTAINED. A derived closure-env FIELD is module-homed (its id derives + ## from the captured local, NOT `@bk` — see itemids.derivedFieldId) so + ## `shouldWriteSymDef` would seal it after the first entry and later entries + ## would reference a def that their indexed copy does not contain. Re-emit it + ## as a full def per entry, deduped within the entry via `emittedFieldSyms`. + w.lowering and sym.kindImpl == skField and not sym.itemId.isBackendMinted + proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = if sym == nil: dest.addDotToken() + elif isLoweredPerEntryField(w, sym): + if not w.emittedFieldSyms.containsOrIncl(sym.itemId): + writeSymDef(w, dest, sym) + else: + dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo elif sym.itemId.isBackendMinted: # Process-local backend sym (closure env field / hidden `:env` param): emit a # MODULE-LOCAL `@bk` def the first time, reference it after. Per-Writer dedup. @@ -556,11 +593,18 @@ proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = # dedup shared with `writeSym`), regardless of module: their itemId.module is # the systemModule of `vmTransfIdgen`, so `shouldWriteSymDef` (which gates on # currentModule) would otherwise only ever emit a SymUse → dangling def. + let perEntryField = isLoweredPerEntryField(w, sym) + # A field reached while writing its own type's reclist MUST be a self-contained + # def: the type can be seek-loaded by name in isolation, so a deduped SymUse + # (whose def lives elsewhere in the entry) would resolve to nil. + let reclistField = w.lowering and w.inTypeReclist > 0 and sym.kindImpl == skField let wantDef = - if sym.itemId.isBackendMinted: not w.emittedBackendSyms.containsOrIncl(sym.itemId.item) + if reclistField: true + elif sym.itemId.isBackendMinted: not w.emittedBackendSyms.containsOrIncl(sym.itemId.item) + elif perEntryField: not w.emittedFieldSyms.containsOrIncl(sym.itemId) else: shouldWriteSymDef(w, sym) if wantDef: - if not sym.itemId.isBackendMinted: sym.state = Sealed + if not sym.itemId.isBackendMinted and not perEntryField and not reclistField: sym.state = Sealed if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): writeType(w, dest, nodeTyp) @@ -674,6 +718,7 @@ var implTag = registerTag("implementation") var reexpModTag = registerTag("reexpmod") var offerTag = registerTag("offer") var typeOfferTag = registerTag("toffer") +var loweredTag = registerTag("lowered") proc registerNifAstTags*() = ## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag` @@ -706,6 +751,7 @@ proc registerNifAstTags*() = reexpModTag = registerTag("reexpmod") offerTag = registerTag("offer") typeOfferTag = registerTag("toffer") + loweredTag = registerTag("lowered") proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n == nil: @@ -1441,7 +1487,7 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; content.addParRi() let m = modname(w.currentModule, w.infos.config) - let nifFilename = AbsoluteFile(m).changeFileExt(".nif") + let nifFilename = AbsoluteFile(m).changeFileExt(".s.nif") let d = completeGeneratedFilePath(config, nifFilename).string var dest = createTokenBuf(600) @@ -1472,6 +1518,71 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; writeImplCookie(config, thisModule, dest, ifaceHex) writeEdgesFile(config, thisModule, implDeps) +proc collectLoweredLocals(w: var Writer; n: PNode) = + ## Record every transform-created local (a `Complete`, non-`@bk` sym owned by + ## the module being serialized) in `w.locals` so it is written SUFFIX-LESS as + ## an inline def. Pre-existing entities (params, result, callees) were sealed + ## before serialization, so they are skipped here and emit as index SymUses. + if n == nil: return + if n.kind == nkSym and n.sym != nil: + let s = n.sym + if not s.itemId.isBackendMinted and s.itemId.module == w.currentModule and + s.state == Complete: + w.locals.incl s.itemId + for i in 0 ..< n.safeLen: + collectLoweredLocals(w, n[i]) + +proc serializeLoweredBodies*(config: ConfigRef; ownerModule: int32; + entries: openArray[tuple[name: string; body: PNode]]; + hooks: openArray[LogEntry]; + outfile: string) = + ## Write the `lower` backend stage's `.t.nif`. Three sections, all covered by + ## the file's embedded index (which the `cg` loader registers as the module's + ## SECOND index/stream — see loadLoweredBodies): + ## - `(repdestroy/repcopy/... key sym)`: the type-bound ops the lower stage + ## lifted while transforming (closure-env `=destroy` etc.). `cg`'s + ## `registerLoadedHooks` re-attaches them so `injectDestructorCalls` (kept in + ## cg) resolves them via `getAttachedOp`'s key fallback. + ## - the hook ROUTINES as full `(sd)`: NEW in the lower stage (no `.s.nif` + ## signature), so serialize sig + transformed body whole; cg loads them via + ## the `.t.nif` index when the HookEntry's SymUse resolves, then demand-emits. + ## - owned routines' transformed bodies as `(lowered "" )`: only + ## the body (signature comes from the `.s.nif`); applyLoweredBodies sets it on + ## the existing sym. + ## A `@bk` closure-env type/sym referenced anywhere is emitted inline AND + ## indexed, so it resolves through the embedded index regardless of section. + var w = Writer(infos: LineInfoWriter(config: config), currentModule: ownerModule) + w.inProc = 1 # we are serializing routine *bodies* + w.lowering = true + var dest = createTokenBuf(256) + createStmtList(dest, NoLineInfo) + for op in hooks: + writeOp(w, dest, op) + var emittedHooks = initHashSet[int32]() + for op in hooks: + if op.sym != nil and op.sym.kindImpl in routineKinds and + not emittedHooks.containsOrIncl(op.sym.itemId.item): + w.emittedBackendTypes.clear() + w.emittedBackendSyms.clear() + w.emittedFieldSyms.clear() + w.locals.clear() + writeSymDef(w, dest, op.sym) + for e in entries: + # Each `(lowered ...)` entry is loaded independently, so it must be + # SELF-CONTAINED: reset the per-Writer backend-local (@bk) dedup so every + # entry re-emits the def of any closure-env type/sym it references. + w.emittedBackendTypes.clear() + w.emittedBackendSyms.clear() + w.emittedFieldSyms.clear() + w.locals.clear() + collectLoweredLocals(w, e.body) + dest.addParLe loweredTag, NoLineInfo + dest.addStrLit e.name + writeNode(w, dest, e.body) + dest.addParRi + dest.addParRi() + writeFile(dest, outfile) + # --------------------------- Loader (lazy!) ----------------------------------------------- proc nodeKind(n: Cursor): TNodeKind {.inline.} = @@ -1533,6 +1644,15 @@ type suffix: string contentStart: int # stream offset of the module body, so a full-AST load can # rewind after lazy symbol loads moved the cursor + # The module's `.t.nif` (the `lower` stage's transformed bodies): a SECOND + # embedded-index + stream consulted when the main `.s.nif` index misses. The + # transformed bodies' backend-minted (`@bk`) entities (closure-env types/syms, + # temporaries) live ONLY here; they are named with this module's suffix, so + # `createTypeStub`/`loadSymStub` look them up here on a `.s.nif` miss and load + # them from `tStream` (see applyLoweredBodies). "Just use NIF's embedded index." + hasTIndex: bool + tStream: nifstreams.Stream + tIndex: Table[string, NifIndexEntry] DecodeContext* = object infos: LineInfoWriter @@ -1568,8 +1688,8 @@ proc loadedState(c: DecodeContext): ItemState {.inline.} = if c.infos.config.cmd == cmdNifC: Complete else: Sealed proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry; - buf: var TokenBuf): Cursor = - let s = addr c.mods[module].stream + buf: var TokenBuf; fromT = false): Cursor = + let s = if fromT: addr c.mods[module].tStream else: addr c.mods[module].stream s.r.jumpTo entry.offset # A seek-load is self-contained: its tokens must decode their relative line # info against `entry.info` ALONE. The stream's `parents` stack can be left at @@ -1631,7 +1751,7 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): # but haven't had their NIF index loaded yet let hasEntry = c.mods.hasKey(result) if not hasEntry or AlwaysLoadInterface in flags: - let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string + let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.nif")).string if not fileExists(modFile): raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile & ". This can happen when loading a module from NIF that references another module " & @@ -1723,10 +1843,17 @@ proc createTypeStub(c: var DecodeContext; t: SymId): PType = let realSuffix = if isBk: suffix[0 ..< suffix.len - BackendLocalMarker.len] else: suffix let modIdx = moduleId(c, realSuffix).int32 let id = if isBk: backendItemId(modIdx, itemVal) else: itemId(modIdx, itemVal) - let ii = addr c.mods[id.module.FileIndex].index - let offs = ii[].getOrDefault(name) + let modFi = id.module.FileIndex + let ii = addr c.mods[modFi].index + var offs = ii[].getOrDefault(name) if offs.offset == 0: - raiseAssert "symbol has no offset: " & name + # A backend-minted (`@bk`) type produced by the `lower` stage lives in this + # module's `.t.nif`, not its `.s.nif` index. Resolve it through the `.t.nif` + # embedded index (loadType picks `tStream` for a tIndex-only name). + if c.mods[modFi].hasTIndex: + offs = c.mods[modFi].tIndex.getOrDefault(name) + if offs.offset == 0: + raiseAssert "symbol has no offset: " & name result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) c.types[name] = (result, offs) @@ -1821,7 +1948,16 @@ proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string; inc val[] let id = if isBk: backendItemId(module.int32, val[]) else: itemId(module.int32, val[]) - let offs = c.mods[module].index.getOrDefault(symAsStr) + var offs = c.mods[module].index.getOrDefault(symAsStr) + if offs.offset == 0 and c.mods[module].hasTIndex: + # A sym produced by the `lower` stage lives in this module's `.t.nif`, not + # its `.s.nif` index. This covers BOTH backend-minted (`@bk`) entities + # (closure `:env` param, `:tmp`/`res` temporaries) AND module-homed derived + # closure-env FIELDS (e.g. a captured `i`): the latter have a normal + # module suffix but are added to the env object only in the backend, so + # they are indexed solely in `.t.nif`. Resolve through that embedded index + # (loadSym picks `tStream` for a tIndex-only name). + offs = c.mods[module].tIndex.getOrDefault(symAsStr) if offs.offset == 0: # Only module/package self-syms are never written as `(sd)` entries, so a # missing index offset means this is such a sym — typically the OWNER of an @@ -1902,7 +2038,12 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms raiseAssert "(td) expected" var scanCursor = n # copy cursor at start of type - let typesModule = parseSymName(pool.syms[n.firstSon.symId]).module + var typesModule = parseSymName(pool.syms[n.firstSon.symId]).module + if typesModule.endsWith(BackendLocalMarker): + # A backend-minted (`@bk`) type's name carries the marker in its module part; + # strip it so the nested-local pre-scan resolves the real module, not a + # nonexistent `@bk.nif`. + typesModule = typesModule[0 ..< typesModule.len - BackendLocalMarker.len] extractLocalSymsFromTree(c, scanCursor, typesModule, localSyms) inc n # move past (td @@ -1941,8 +2082,22 @@ proc loadType*(c: var DecodeContext; t: PType) = if t.state != Partial: return t.state = c.loadedState var buf = createTokenBuf(30) - let typeName = typeToNifSym(t, c.infos.config) - var n = cursorFromIndexEntry(c, t.itemId.module.FileIndex, c.types[typeName][1], buf) + # A backend-minted (`@bk`) closure-env type produced by the `lower` stage lives + # ONLY in the `.t.nif` and is keyed by its `@bk` name (see nifTypeName), not the + # canonical `typeToNifSym` (which asserts non-`@bk`). Reconstruct that name so a + # Partial `@bk` stub that escaped the inline pre-scan can still be force-loaded. + let typeName = + if t.uniqueId.isBackendMinted: + "`t" & $ord(t.kind) & "." & $t.uniqueId.item & "." & + modname(t.itemId.module, c.infos.config) & BackendLocalMarker + else: + typeToNifSym(t, c.infos.config) + let modFi = t.itemId.module.FileIndex + # A name resolved through the `.t.nif` (tIndex) — a `lower`-stage closure-env + # type — must seek in `tStream`, not the `.s.nif` stream. + let fromT = c.mods[modFi].hasTIndex and not c.mods[modFi].index.hasKey(typeName) and + c.mods[modFi].tIndex.hasKey(typeName) + var n = cursorFromIndexEntry(c, modFi, c.types[typeName][1], buf, fromT = fromT) var localSyms = initTable[string, PSym]() loadTypeFromCursor(c, n, t, localSyms) @@ -2027,6 +2182,15 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: loadLoc c, n, s.locImpl s.constraintImpl = loadNode(c, n, thisModule, localSyms) s.instantiatedFromImpl = loadSymStub(c, n, thisModule, localSyms) + # The TRANSFORMED body slot (see writeSymDef). Reconstruct it ONLY in the + # backend (`cmdNifC`), where `transformBody` short-circuits on it; during + # frontend sem (`cmdM`) skip the tokens — a dependent never needs a foreign + # routine's lowered body, and reconstructing one must not perturb effect/ + # exception inference (the "never change frontend node-typing for IC" rule). + if c.infos.config.cmd == cmdNifC and s.kindImpl in routineKinds: + s.transformedBodyImpl = loadNode(c, n, thisModule, localSyms) + else: + skip n skipParRi n proc loadSym*(c: var DecodeContext; s: PSym) = @@ -2035,7 +2199,10 @@ proc loadSym*(c: var DecodeContext; s: PSym) = var buf = createTokenBuf(30) let symsModule = s.itemId.module.FileIndex let nifname = globalName(s, c.infos.config) - var n = cursorFromIndexEntry(c, symsModule, c.syms[nifname][1], buf) + # A `@bk` sym resolved through the `.t.nif` (tIndex) seeks in `tStream`. + let fromT = c.mods[symsModule].hasTIndex and not c.mods[symsModule].index.hasKey(nifname) and + c.mods[symsModule].tIndex.hasKey(nifname) + var n = cursorFromIndexEntry(c, symsModule, c.syms[nifname][1], buf, fromT = fromT) expect n, ParLe if n.tagId != sdefTag: @@ -2053,6 +2220,149 @@ proc loadSym*(c: var DecodeContext; s: PSym) = inc n loadSymFromCursor(c, s, n, c.mods[symsModule].suffix, localSyms) +proc sealLoadedBackendEntities*(c: var DecodeContext) = + ## Before the `lower` stage serializes its transformed bodies, mark every + ## index-loaded sym/type `Sealed`. The backend loads them `Complete` (mutable + ## for the transform); without this, `writeNode`'s `shouldWriteSymDef` would + ## emit a duplicate `(sd)`/`(td)` def for a param/result/existing-local/owner + ## type — and the `cg` body-loader would then bind the body's `result` to a + ## FRESH sym instead of the one in `prc.ast[resultPos]` (the #6/#7 "result + ## cannot be captured" class). Sealed ⟹ SymUse ⟹ resolved via `cg`'s module + ## index. The transform-CREATED entities are absent from `c.syms`/`c.types` + ## (or are backend-minted), so they stay `Complete`/`@bk` and still get the + ## inline defs the loader's local-sym pre-scan needs. + for _, v in c.syms: + if v[0] != nil and v[0].state == Complete: v[0].state = Sealed + for _, v in c.types: + if v[0] != nil and v[0].state == Complete: v[0].state = Sealed + +proc preloadLoweredDefs(c: var DecodeContext; n: var Cursor; thisModule: string; + localSyms: var Table[string, PSym]) = + ## One pre-scan over a `.t.nif` body that fully loads EVERY inline def it + ## carries before the body is decoded: suffix-less locals into `localSyms`, + ## backend-minted `@bk` syms into `c.syms`, `@bk` types into `c.types`. These + ## transform-created entities have NO module index entry, so a later reference + ## — in the body, or inside a loaded type that `typeKey`/codegen later walks + ## (where the index-based force-load would assert) — must find them already + ## loaded. The writer emits defs before uses, so a forward walk that loads each + ## in place suffices. + if n.kind != ParLe: + inc n + return + var depth = 0 + while true: + if n.kind == ParLe: + if n.tagId == sdefTag: + let nm = n.firstSon + if nm.kind == SymbolDef: + let symName = pool.syms[nm.symId] + let sn = parseSymName(symName) + if sn.module.len == 0: + if symName notin localSyms: + let module = moduleId(c, thisModule) + let val = addr c.mods[module].symCounter + inc val[] + let sym = PSym(itemId: itemId(module.int32, val[]), kindImpl: skStub, + name: c.cache.getIdent(sn.name), disamb: sn.count.int32, + state: Complete) + localSyms[symName] = sym + inc n + loadSymFromCursor(c, sym, n, thisModule, localSyms) + sym.state = c.loadedState + continue + elif sn.module.endsWith(BackendLocalMarker): + let sym = c.loadSymStub(nm.symId, thisModule, localSyms) + if sym.state == Partial: + sym.state = c.loadedState + inc n + loadSymFromCursor(c, sym, n, thisModule, localSyms) + continue + elif n.tagId == tdefTag: + let nm = n.firstSon + if nm.kind == SymbolDef and pool.syms[nm.symId].endsWith(BackendLocalMarker): + discard loadTypeStub(c, n, localSyms) + continue + inc depth + elif n.kind == ParRi: + dec depth + if depth == 0: + inc n + break + inc n + +proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym + +proc repTagToOp(tagId: TagId): (bool, TTypeAttachedOp) = + ## Map a `(rep…)` hook tag to its attached-op kind (and whether it IS one). + if tagId == repDestroyTag: (true, attachedDestructor) + elif tagId == repCopyTag: (true, attachedAsgn) + elif tagId == repWasMovedTag: (true, attachedWasMoved) + elif tagId == repDupTag: (true, attachedDup) + elif tagId == repSinkTag: (true, attachedSink) + elif tagId == repTraceTag: (true, attachedTrace) + elif tagId == repDeepCopyTag: (true, attachedDeepCopy) + else: (false, attachedDestructor) + +proc loadLoweredBodies*(c: var DecodeContext; module: FileIndex; suffix: string; + infile: string; loadBodies = true): + tuple[bodies: seq[tuple[name: string; body: PNode]]; hooks: seq[LogEntry]] = + ## Reconstruct the `lower` stage's `.t.nif`. Registers its embedded index as the + ## module's SECOND index/stream (`tIndex`/`tStream`) so every backend-minted + ## (`@bk`) closure-env entity resolves through NIF's own index — `createTypeStub`/ + ## `loadSymStub` fall through to it on a `.s.nif` miss, `loadType`/`loadSym` seek + ## `tStream`. Returns the owned routines' transformed bodies (the `(lowered …)` + ## entries) and the lifted type-bound ops (`(rep… key sym)`); the hook ROUTINES' + ## `(sd)` defs are loaded lazily through the index when their op's sym resolves. + result = (@[], @[]) + if not fileExists(infile): return + var tstream = nifstreams.open(infile) + let tindex = readEmbeddedIndex(tstream) # leaves the cursor at the content start + let contentStart = offset(tstream.r) + c.mods[module].tStream = tstream + c.mods[module].tIndex = tindex + c.mods[module].hasTIndex = true + # Parse the WHOLE content up front: the per-body loads below lazily seek + # `tStream` for `@bk` entities, which would otherwise clobber a live walk cursor. + var buf = createTokenBuf(256) + tstream.r.jumpTo contentStart + nifcursors.parse(tstream, buf, NoLineInfo) + var n = beginRead(buf) + if n.kind != ParLe: return + inc n # into (stmts -> flags dot + inc n # -> type dot + inc n # -> first entry or ParRi + while n.kind == ParLe: + if n.tagId == loweredTag: + if not loadBodies: + # A dependency: register its `.t.nif` (tIndex + hooks) so the consumer + # can destroy the dep's closures, but don't reconstruct its bodies (only + # the dep's OWN cg emits them). + skip n + continue + inc n # -> StringLit name + let name = pool.strings[n.litId] + inc n # -> body tree + var localSyms = initTable[string, PSym]() + var scanCursor = n + extractLocalSymsFromTree(c, scanCursor, suffix, localSyms) + let body = loadNode(c, n, suffix, localSyms) + result.bodies.add (name, body) + skipParRi n # close (lowered ...) + else: + let (isHook, op) = repTagToOp(n.tagId) + if isHook: + inc n # -> StringLit key + let key = pool.strings[n.litId] + inc n # -> Symbol sym + let sym = resolveHookSym(c, n.symId) + inc n + if sym != nil: + result.hooks.add LogEntry(kind: HookEntry, op: op, module: module.int, + key: key, sym: sym) + skipParRi n + else: + skip n # a hook routine's `(sd)` def — loaded lazily via the index + template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info) @@ -2126,9 +2436,47 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; loadSymFromCursor(c, sym, n, thisModule, localSyms) sym.state = c.loadedState # mark as fully loaded result = newSymNode(sym, info) - else: + elif sn.module.endsWith(BackendLocalMarker): + # A backend-minted (`@bk`) def lives ONLY inline in this `.t.nif` body + # (not in any module index): create/find its cached stub and FILL it + # from the sdef instead of skipping (which would leave the skModule/ + # Partial stub `loadSymStub` made unresolved). sym = c.loadSymStub(name.symId, thisModule, localSyms) - skip n # skip the entire sdef for indexed symbols + if sym.state == Partial: + sym.state = c.loadedState + inc n # skip `sd` tag + loadSymFromCursor(c, sym, n, thisModule, localSyms) + else: + skip n + result = newSymNode(sym, info) + result.flags.incl nfLazyType + else: + # A module-homed inline sdef. Normally its def lives in that module's + # index and is loaded lazily, so we skip the inline copy. BUT a + # transform-created closure-env FIELD (`x0.0.clo`) is module-homed yet + # lives ONLY inline in this `.t.nif` reclist — it has no index entry. + # Skipping it leaves a nil-typed `skModule` fallback stub (from + # loadSymStub's "no offset" path) and codegen of the env struct then + # dereferences a nil field type. Detect the unindexed case and FILL + # the sym from the inline def instead. + let m = moduleId(c, sn.module) + let indexed = c.mods[m].index.hasKey(symName) or + (c.mods[m].hasTIndex and c.mods[m].tIndex.hasKey(symName)) + if indexed: + sym = c.loadSymStub(name.symId, thisModule, localSyms) + skip n # skip the entire sdef for indexed symbols + else: + sym = c.syms.getOrDefault(symName)[0] + if sym == nil: + let val = addr c.mods[m].symCounter + inc val[] + sym = PSym(itemId: itemId(m.int32, val[]), kindImpl: skStub, + name: c.cache.getIdent(sn.name), disamb: sn.count.int32, + state: Partial) + c.syms[symName] = (sym, NifIndexEntry()) + sym.state = c.loadedState + inc n # skip `sd` tag + loadSymFromCursor(c, sym, n, thisModule, localSyms) result = newSymNode(sym, info) result.flags.incl nfLazyType of typeDefTagName: @@ -2289,7 +2637,7 @@ proc moduleSymbolStubs*(c: var DecodeContext; module: FileIndex): seq[PSym] = proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) - result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string + result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.nif").string proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym = result = c.syms.getOrDefault(symAsStr)[0] @@ -2311,6 +2659,11 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo # Try the format without module suffix let localKey = sn.name & "." & $sn.count & "." offs = c.mods[module].index.getOrDefault(localKey) + if offs.offset == 0 and c.mods[module].hasTIndex: + # A `lower`-stage entity (an `@bk` hook routine, OR a module-homed derived + # closure-env field added only in the backend) lives in the module's + # `.t.nif`, not its `.s.nif`: resolve it through the second (tIndex) index. + offs = c.mods[module].tIndex.getOrDefault(symAsStr) if offs.offset == 0: return nil if not alsoConsiderPrivate and offs.vis == Hidden: diff --git a/compiler/cnif.nim b/compiler/cnif.nim index 0caebd8682..ad51f47f8c 100644 --- a/compiler/cnif.nim +++ b/compiler/cnif.nim @@ -358,6 +358,52 @@ proc readCnifHeads*(f: string): CnifHeads = endRead(c) result.valid = sawMeta and version == CnifVersion +proc writeLoweredArtifact*(outfile: string; entries: openArray[string]) = + ## The `.t.nif` "lowered" artifact: one `(lowered "" )` per + ## routine the module OWNS, written by the per-module `lower` backend stage + ## for the `cg` stage to read instead of re-deriving the transformed body + ## (re-derivation in each parallel `cg` process is what makes a closure + ## `:env`'s identity diverge across modules — see transf.transformBody). + ## + ## SKELETON: every body is the empty-marker `.` ("transformed body == sem + ## body"), so `cg` falls back to its own `transformBody` and output stays + ## byte-identical. The real transformed body fills this slot in a later step; + ## the `.` then means "unchanged by lowering" (the dedup Araq sketched). + var b = nifbuilder.open(outfile) + b.withTree "stmts": + for name in entries: + b.withTree "lowered": + b.addStrLit name + b.addEmpty() + b.close() + +proc readLoweredArtifact*(f: string): seq[string] = + ## The routine NIF names recorded in a `.t.nif`. (Bodies are not returned + ## yet: the skeleton records only empty-markers; reading proves the artifact + ## round-trips and that the `cg` rule depends on it.) + result = @[] + if not fileExists(f): return + var pool = newPool() + var tags = newTagPool() + let stmtsTag = tags.registerTag("stmts") + let loweredTag = tags.registerTag("lowered") + var buf = parseFromFile(f, 1000, pool, tags) + var c = beginRead(buf) + if c.kind != TagLit or c.cursorTagId != stmtsTag: + endRead(c) + return + c.loopInto: + if c.kind == TagLit and c.cursorTagId == loweredTag: + c.loopInto: + if c.kind == StrLit: + result.add strVal(c) + inc c + else: + skip c + else: + skip c + endRead(c) + type CnifLiveness* = object defs*: int ## proc definitions emitted across all modules diff --git a/compiler/deps.nim b/compiler/deps.nim index 08a272aa6b..3dec7fc507 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -51,7 +51,7 @@ proc parsedFile(c: DepContext; f: FilePair): string = getNimcacheDir(c.config).string / f.modname & ".p.nif" proc semmedFile(c: DepContext; f: FilePair): string = - getNimcacheDir(c.config).string / f.modname & ".nif" + getNimcacheDir(c.config).string / f.modname & ".s.nif" proc ifaceFile(c: DepContext; f: FilePair): string = ## Interface-cookie sidecar written by `nim m` (ast2nif.writeIfaceCookie, @@ -941,9 +941,11 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string # Per-node output paths. var cnifFiles = newSeq[string](c.nodes.len) var cFiles = newSeq[string](c.nodes.len) + var tFiles = newSeq[string](c.nodes.len) for i, node in c.nodes: cFiles[i] = backendCFile(c, node) cnifFiles[i] = cFiles[i] & ".nif" + tFiles[i] = cFiles[i] & ".t.nif" var b = nifbuilder.open(result) defer: b.close() @@ -979,9 +981,28 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string b.addStrLit s b.endTree() - # cg: one rule per module. Inputs are the project (slot 0) and every semmed - # NIF (so the whole program loads and the rule is ordered after the frontend); - # the main module additionally depends on every other `.c.nif` (init metas). + # lower: one rule per module. Transforms (eventually) the routines the module + # OWNS once, in the owner's id space, into `.t.nif`, so the `cg` stage + # reads them instead of re-deriving (which makes a closure `:env`'s identity + # diverge across the parallel `cg` processes). Runs per module in parallel on + # the shallow backend dep-graph. Inputs mirror `cg` (project + every semmed + # NIF) so the rule is ordered after the frontend. + for i, node in c.nodes: + b.addTree "do" + b.addIdent "nim_nifc" + b.withTree "args": + b.addStrLit "--icBackendStage:lower" + b.addStrLit "--icBackendModule:" & node.files[0].modname + inputStr mainNif + for n2 in c.nodes: + inputStr c.semmedFile(n2.files[0]) + outputStr tFiles[i] + b.endTree() + + # cg: one rule per module. Inputs are the project (slot 0), every semmed + # NIF (so the whole program loads and the rule is ordered after the frontend) + # and this module's `.t.nif` (its lowered bodies); the main module additionally + # depends on every other `.c.nif` (init metas). for i, node in c.nodes: b.addTree "do" b.addIdent "nim_nifc" @@ -991,6 +1012,7 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string inputStr mainNif for n2 in c.nodes: inputStr c.semmedFile(n2.files[0]) + inputStr tFiles[i] if node.id == 0: for j in 0 ..< c.nodes.len: if c.nodes[j].id != 0: diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 2b5ae6421e..7f5b00ee8b 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -245,6 +245,18 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode let canon = c.graph.canonTypes.getOrDefault(h) if canon != nil: op = getAttachedOp(c.graph, canon, kind) + if (op == nil or op.ast.isGenericRoutine) and icLoweredBodies(c.graph.config): + # IC: injectDestructorCalls is demand-driven and runs HERE (cg), not in the + # `lower` stage, so a structural, env-agnostic op the lower stage never had + # reason to serialize — most often a closure PROC type's `=destroy`/`=sink` + # (which act on the `(ClP_0, ClE_0)` tuple, NOT the concrete env) — must be + # lifted on demand, exactly as the lazy path's cg does. This is safe now: + # closure-env identity resolves via `attachedOps[itemId]`/env-erased typeKey, + # env objects load complete, and atomicRefOp's type-erased path covers any + # still-incomplete env (so the lift never walks a nil field). + excl t.flagsImpl, tfCheckedForDestructor + createTypeBoundOps(c.graph, nil, t, dest.info, c.idgen) + op = getAttachedOp(c.graph, t, kind) if op == nil: #echo dest.typ.id globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] & diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 9e00179fbb..3418bad729 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -176,7 +176,7 @@ proc closureParams(routine: PSym): PNode = result = routine.typ.n routine.ast[paramsPos] = result -proc addHiddenParam(routine: PSym, param: PSym) = +proc addHiddenParam*(routine: PSym, param: PSym) = assert param.kind == skParam var params = closureParams(routine) # -1 is correct here as param.position is 0 based but we have at position 0 diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 3684ec437a..60408a5861 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -761,6 +761,28 @@ proc cyclicType*(g: ModuleGraph, t: PType): bool = of tyProc: result = t.callConv == ccClosure else: result = false +proc recHasNilFieldType(n: PNode): bool = + case n.kind + of nkSym: result = n.sym == nil or n.sym.typ == nil + of nkRecList, nkRecCase: + for ch in n: + if recHasNilFieldType(ch): return true + result = false + else: result = false + +proc isTypeErasedEnvRef(config: ConfigRef; elemType: PType): bool = + ## IC: a foreign closure-env object can load (cross-module) with nil-typed + ## derived fields when its hook key diverges across the NIF boundary. Per the + ## closure type-erasure principle, destroying such a `ref` must go through RTTI + ## (`nimDestroyAndDispose` / `nimDecRefIsLastCyclicDyn`), NEVER a statically + ## lifted concrete env destructor — the producer module emits that destructor + ## and registers it in the env object's type info. Detect the incomplete load + ## so `atomicRefOp` takes the dynamic-dispatch path and never walks the nil + ## field (which would SIGSEGV). + if not config.icLoweredBodies: return false + let obj = elemType.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}) + result = obj.kind == tyObject and obj.n != nil and recHasNilFieldType(obj.n) + proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = #[ bug #15753 is really subtle. Usually the classical write barrier for reference counting looks like this:: @@ -793,13 +815,17 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = ]# var actions = newNodeI(nkStmtList, c.info) let elemType = t.elementType + # (b) type-erasure: a foreign closure env that loaded incomplete cross-module + # must be destroyed via RTTI, not by lifting its concrete (nil-fielded) object. + let erased = isTypeErasedEnvRef(c.g.config, elemType) - createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen) + if not erased: + createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen) # YRC uses dedicated runtime procs for the entire write barrier: if c.g.config.selectedGC == gcYrc: let desc = - if isFinal(elemType): + if isFinal(elemType) and not erased: let ti = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) ti.typ = getSysType(c.g, c.info, tyPointer) ti @@ -814,31 +840,40 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = return else: discard # fall through for destructor, trace, wasMoved - let isCyclic = c.g.config.selectedGC in {gcOrc, gcYrc} and types.canFormAcycle(c.g, elemType) + # `erased` must short-circuit BEFORE canFormAcycle/isPureObject/isFinal, which + # walk the (nil-fielded) type graph. Assume cyclic + dynamic — the dyn runtime + # calls work for any ref via its type info. + let isCyclic = erased or + (c.g.config.selectedGC in {gcOrc, gcYrc} and types.canFormAcycle(c.g, elemType)) - let isInheritableAcyclicRef = c.g.config.selectedGC in {gcOrc, gcYrc} and + let isInheritableAcyclicRef = (not erased) and c.g.config.selectedGC in {gcOrc, gcYrc} and (not isPureObject(elemType)) and tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags # dynamic Acyclic refs need to use dyn decRef + let useStatic = (not erased) and isFinal(elemType) + let tmp = if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}: declareTempOf(c, body, x) else: x - if isFinal(elemType): + if useStatic: addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr)) var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType)) alignOf.typ = getSysType(c.g, c.info, tyInt) actions.add callCodegenProc(c.g, "nimRawDispose", c.info, tmp, alignOf) else: - addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(tmp, nkDerefExpr)) + # `nimDestroyAndDispose` resolves the real destructor via the object's RTTI, + # so the env destructor the producer emitted runs — no static lift needed. + if not erased: + addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(tmp, nkDerefExpr)) actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, tmp) var cond: PNode if isCyclic: - if isFinal(elemType): + if useStatic: let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) typInfo.typ = getSysType(c.g, c.info, tyPointer) cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo) @@ -873,7 +908,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: if isCyclic: - if isFinal(elemType): + if useStatic: let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) typInfo.typ = getSysType(c.g, c.info, tyPointer) body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 98f7b58537..65c8a070f3 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -904,7 +904,7 @@ proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} = assert result != nil when not defined(nimKochBootstrap): - proc registerLoadedHooks(g: ModuleGraph; logOps: seq[LogEntry]) = + proc registerLoadedHooks*(g: ModuleGraph; logOps: seq[LogEntry]) = let mainSuffix = getMainModuleSuffix(ast.program) for x in logOps: # A dependency's NIF may carry hooks whose syms belong to the module we diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index ed2567d91c..b83e25e152 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -26,6 +26,8 @@ import ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif, typekeys, cnif from cgmeth import generateIfMethodDispatchers +from transf import transformBody +from lambdalifting import getEnvParam, addHiddenParam, paramName import ic / replayer proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex; @@ -145,6 +147,32 @@ proc signatureHasMetaType(t: PType; depth: int = 0): bool = for k in t.kids: if signatureHasMetaType(k, depth + 1): return true +proc ownsRuntimeRoutine(s: PSym; modPos: int): bool = + ## A concrete, non-generic, runtime routine with a real body, OWNED by the + ## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a + ## routine called only from other modules is still emitted by somebody) and + ## the `lower` stage's owned-routine enumeration, so both stages see exactly + ## the same set. The exclusions: + ## - nested/closure procs (owner is a proc, not a module): emitted via their + ## enclosing routine's lambda-lifting, never standalone; + ## - generic instances (`sfFromGeneric`): emitted by demand, deduped by merge; + ## - `importc`/`compileTime`/`error`/forward sentinels and meta signatures: + ## not real codegen targets. + ## A `{.closure.}` iterator IS a standalone runtime routine (unlike an inline + ## iterator, which is expanded at each call site) and must be emitted by its + ## owner — else a cross-module `for` over it links to nothing. + s.itemId.module == modPos and + (s.kind in {skProc, skFunc, skConverter, skMethod} or + (s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and + s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and + s.magic == mNone and + sfFromGeneric notin s.flags and + {sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and + s.typ != nil and not signatureHasMetaType(s.typ) and + s.ast != nil and s.ast.safeLen > bodyPos and + s.ast[genericParamsPos].kind == nkEmpty and + s.ast[bodyPos].kind != nkEmpty + proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) = ## Generate C code for a single module. let moduleId = precomp.module.position @@ -170,34 +198,7 @@ proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) = if g.config.cmd == cmdNifC and g.config.icBackendStage == "cg": let modPos = precomp.module.position for s in moduleSymbolStubs(ast.program, FileIndex modPos): - if s.itemId.module == modPos and - s.kind in {skProc, skFunc, skConverter, skMethod} and - # Only MODULE-level routines: a nested/closure proc (its owner is a - # proc) captures its enclosing scope and cannot be emitted standalone — - # the captured params have no loc → `expr: param not init`. Nested procs - # are emitted via their enclosing routine's lambda-lifting, so seeding - # the enclosing (module-level) routine already covers them. - s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and - s.magic == mNone and - # Skip generic instances: they have no single owning-module top-level - # and are emitted by demand (emit-everywhere, deduped by the merge - # stage). An instance has an empty `genericParamsPos` just like a plain - # concrete proc, so only `sfFromGeneric` tells them apart; seeding one - # would force standalone codegen of an instance body whose `when T is X` - # branches were never folded for this path → `genMagicExpr: mIs`. - sfFromGeneric notin s.flags and - # Every other routine the module owns must be emitted here, exported or - # not: a non-exported helper is still reached from another module when a - # `template`/inline routine expands at a call site there (e.g. msgs' - # `internalErrorImpl` behind the `internalError` template), and that - # caller now only prototypes it. `{.error.}`/`compileTime` sentinels and - # bodyless forward decls are not real codegen targets. - {sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and - s.typ != nil and not signatureHasMetaType(s.typ) and - s.ast != nil and s.ast.safeLen > bodyPos and - s.ast[genericParamsPos].kind == nkEmpty and - s.ast[bodyPos].kind != nkEmpty: - # a concrete, non-generic, runtime routine with a real body, owned here + if ownsRuntimeRoutine(s, modPos): requestProcDef(bmod, s) proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex): @@ -323,6 +324,218 @@ proc findTargetModule(g: ModuleGraph; modules: seq[PrecompiledModule]; cachedModuleSuffix(g.config, FileIndex precompSys.module.position) == suffix: return precompSys +proc findHiddenEnvParam(n: PNode; owner: PSym): PSym = + ## Locate the hidden env param (`:envP`) that belongs to `owner` in its loaded + ## transformed body, so it can be re-welded into `owner`'s signature. Matching + ## by `owner` is essential: a body can ALSO reference a callee's `:envP` (a + ## closure call passes the callee env), so the first `:envP` in DFS order is + ## not necessarily this proc's own. + if n == nil: return nil + if n.kind == nkSym: + if n.sym != nil and n.sym.kind == skParam and n.sym.name.s == paramName and + n.sym.owner == owner: + return n.sym + else: + for i in 0 ..< n.safeLen: + let r = findHiddenEnvParam(n[i], owner) + if r != nil: return r + return nil + +proc registerLoweredModule(g: ModuleGraph; m: PSym; applyBodies: bool) = + ## Load module `m`'s `.t.nif` and register its `lower`-stage output: its + ## embedded index (so its closure-env `@bk` entities resolve) and its lifted + ## type-bound ops (so any cg that DESTROYS one of `m`'s closures finds the env + ## `=destroy` via getAttachedOp). For the cg TARGET (`applyBodies`), also set + ## each owned routine's `transformedBody` so cg's `transformBody` short-circuits + ## (transf.nim:1383) instead of re-deriving. `injectDestructorCalls` stays in cg. + let modPos = m.position + let bmod = BModuleList(g.backend).mods[modPos] + if bmod == nil: return + let artifact = getCFile(bmod).string & ".t.nif" + if not fileExists(artifact): return + let suffix = cachedModuleSuffix(g.config, FileIndex modPos) + let (bodies, hooks) = loadLoweredBodies(ast.program, FileIndex modPos, suffix, + artifact, loadBodies = applyBodies) + registerLoadedHooks(g, hooks) + if not applyBodies: return + var byName = initTable[string, PSym]() + for s in moduleSymbolStubs(ast.program, FileIndex modPos): + # Owned routines AND nested closure routines (the `:anonymous` procs the + # lower stage emits as their own entries) — both are index-resolvable syms + # of this module whose transformed body the lower stage authored. + if s.kind in routineKinds and s.itemId.module == modPos: + byName[globalName(s, g.config)] = s + for (name, body) in bodies: + let s = byName.getOrDefault(name) + # `.s.nif` wins: only fill from `.t.nif` if sem did not already transform it. + if s != nil and body != nil and s.transformedBody == nil: + s.transformedBody = body + # Lambda-lift in the lower stage gave this proc a hidden `:envP` env param + # (a captured-var closure env), but cg loaded the PRE-lift signature from + # `.s.nif`. The transformed body references that `@bk` `:envP`; re-weld it + # into the proc's params so genProc assigns it a loc (else "param not + # init"). `transformBody` short-circuits on the cached body, skipping the + # lift that normally adds it. This applies to BOTH a true `ccClosure` proc + # (env arrives via the closure ABI `ClE_0`, needs `tfCapturesEnv`) and a + # plain nested `nimcall` proc that merely captures (env is a regular last + # param). Match the env param by owner — a body can also reference a + # callee's `:envP`. + if s.typ != nil and getEnvParam(s) == nil: + let ep = findHiddenEnvParam(body, s) + if ep != nil: + # From-source, `ast[paramsPos]` and `typ.n` are the SAME node, but a + # NIF-loaded routine has two distinct param nodes. genProc reads + # `typ.n`, so unify them first — else addHiddenParam appends to + # `ast[paramsPos]` and the env param never reaches genProc's loc setup. + if s.typ.n != nil: + s.ast[paramsPos] = s.typ.n + addHiddenParam(s, ep) + # The lower stage's lambda-lift converts EVERY captured nested proc to a + # closure (collectNestedClosureBodies only emits `ccClosure` entries), + # and the serialized call sites use the closure ABI. cg loaded the + # pre-lift signature, which for a proc only ever CALLED (never used as a + # value) is still `nimcall`. Re-apply the lift's `ccClosure` + + # `tfCapturesEnv` so closureSetup maps the env param to `ClE_0` and the + # calls match. + s.typ.callConv = ccClosure + incl(s.typ, {tfCapturesEnv}) + +proc applyLoweredBodies(g: ModuleGraph; modules: seq[PrecompiledModule]; + precompSys: PrecompiledModule; target: PrecompiledModule) = + ## Register every loaded module's `.t.nif` (env entities + lifted hooks), + ## applying transformed bodies only for the cg target. + if not icLoweredBodies(g.config): return # Stage 0 (lazy): nothing to apply + if precompSys.module != nil: + registerLoweredModule(g, precompSys.module, applyBodies = false) + for m in modules: + if m.module != nil: + registerLoweredModule(g, m.module, + applyBodies = (m.module.position == target.module.position)) + +proc collectNestedClosureBodies(g: ModuleGraph; idgen: IdGenerator; n: PNode; + owner: PSym; seen: var IntSet; + entries: var seq[tuple[name: string; body: PNode]]) = + ## A closure routine nested in `owner` (the `:anonymous` proc lambda-lifting + ## minted, plus any deeper nesting) gets its captured-var→env rewrite produced + ## as part of the OWNER's `transformBody`, but only the owner's body is emitted + ## as a `(lowered)` entry. The nested proc itself IS index-resolvable (it has a + ## `.s.nif` sdef from sem, with its PRE-lift body), so cg loads that and + ## re-derives — and the capture mapping is gone (it accesses `x` directly + ## instead of `ClE_0->x0`). Walk the transformed body and emit each nested + ## closure routine's transformed body as its OWN `(lowered)` entry so + ## applyLoweredBodies installs it and cg reuses it verbatim. + if n == nil: return + if n.kind == nkSym: + let s = n.sym + if s != nil and s.kind in routineKinds and s != owner and + not seen.containsOrIncl(s.id): + if s.ast != nil and getBody(g, s).kind != nkEmpty and + s.typ != nil and s.typ.callConv == ccClosure: + if s.transformedBody == nil: + s.transformedBody = transformBody(g, idgen, s, {}) + entries.add (globalName(s, g.config), s.transformedBody) + collectNestedClosureBodies(g, idgen, s.transformedBody, s, seen, entries) + else: + for i in 0 ..< n.safeLen: + collectNestedClosureBodies(g, idgen, n[i], owner, seen, entries) + +proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) = + ## Per-module backend lowering (`--icBackendStage:lower --icBackendModule:`): + ## enumerate the routines this module OWNS and write them to `.t.nif`. + ## Eventually this transforms each owned routine once, in the owner's id space, + ## so `cg` reads the result instead of re-deriving it (re-derivation per + ## parallel `cg` process is the root of the closure-`:env` identity drift). + ## Runs per module in parallel on the shallow backend dep-graph — NOT folded + ## into the dense, mostly-serial sem stage. + ## + ## gate `newSymNode`'s lazy-type marking to the backend (see astdef) — the + ## transform builds sym nodes off not-yet-typed stubs, exactly as the `cg` + ## stage does. + nifcBackendActive = true + let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx) + let targetIsMain = g.config.icBackendModule.len == 0 or + g.config.icBackendModule == mainSuffix + var modules: seq[PrecompiledModule] + var precompSys: PrecompiledModule + var target: PrecompiledModule + if targetIsMain: + var nifFiles: seq[string] + (modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx) + if modules.len == 0: + rawMessage(g.config, errGenerated, + "Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx)) + return + target = findTargetModule(g, modules, precompSys, g.config.icBackendModule) + else: + (modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule) + if target.module == nil: + rawMessage(g.config, errGenerated, + "per-module lowering: module not found for suffix: " & g.config.icBackendModule) + return + let modPos = target.module.position + let tb = BModuleList(g.backend).mods[modPos] + if tb == nil: + rawMessage(g.config, errGenerated, + "per-module lowering: no backend module for suffix: " & g.config.icBackendModule) + return + let artifact = getCFile(tb).string & ".t.nif" + if icLoweredBodies(g.config): + # STAGE 1 (DEFAULT; `-d:icNoLowerBodies` opts out): transform every owned routine + # ONCE in this single process's id space and serialize the results, so `cg` + # reads them instead of re-deriving (the single-writer-per-owner that keeps + # closure-`:env` identity stable). `transformBody` with flags {} mirrors the + # cg call (cgen.nim:1409); we keep only its return value (it clears + # `transformedBody` for non-cached procs). `injectDestructorCalls` is NOT run + # — it stays in `cg` on the loaded body. + var entries: seq[tuple[name: string; body: PNode]] = @[] + # `transformBody`/lambda-lifting LIFTS the closure env's type-bound ops + # (`=destroy` etc.) into `g.opsLog`; snapshot its length so we can serialize + # exactly the ops THIS stage created (not those loaded from `.s.nif`). + let opsLogStart = g.opsLog.len + for s in moduleSymbolStubs(ast.program, FileIndex modPos): + if ownsRuntimeRoutine(s, modPos): + # `.s.nif` wins: a routine already transformed during sem (CT eval / + # macro / VM transform) carries its lowered body in the `.s.nif` slot — + # don't re-transform it here, just leave its `.t.nif` entry empty. + if s.transformedBody != nil: continue + let tbody = transformBody(g, tb.idgen, s, {}) + entries.add (globalName(s, g.config), tbody) + var seenNested = initIntSet() + collectNestedClosureBodies(g, tb.idgen, tbody, s, seenNested, entries) + # Collect the hooks this stage lifted, and transform each hook ROUTINE's body + # too (it is itself lowered into NIFC). The hooks' `(sd)` + transformed body go + # into the `.t.nif`; `cg` re-attaches them so `injectDestructorCalls` resolves + # the loaded env's `=destroy`. Iterate to a fixpoint: a hook body can lift + # further hooks (a field's `=destroy`). + var hooks: seq[LogEntry] = @[] + var i = opsLogStart + while i < g.opsLog.len: + let e = g.opsLog[i] + if e.kind == HookEntry and e.sym != nil and e.sym.kind in routineKinds and + e.sym.transformedBody == nil: + hooks.add e + # Transform the hook routine's body and cache it on the sym so + # `writeSymDef` serializes it in the hook's `(sd)` transformed-body slot + # (`transformBody {}` returns the body but does not cache it). + e.sym.transformedBody = transformBody(g, tb.idgen, e.sym, {}) + inc i + # Seal the index-loaded entities so their references in the bodies serialize + # as SymUses (resolved via the module index in cg), not duplicate defs. + sealLoadedBackendEntities(ast.program) + serializeLoweredBodies(g.config, modPos.int32, entries, hooks, artifact) + if isDefined(g.config, "icDceCheck"): + stderr.writeLine "[icLower] " & extractFilename(artifact) & " " & + $entries.len & " routines transformed, " & $hooks.len & " hooks" + else: + # DEFAULT (Stage 0, byte-neutral): record one empty-marker per owned routine. + # `cg` derives the transformed body itself, so output is unchanged; this only + # exercises the artifact + scheduling the transform-move builds on. + var names: seq[string] = @[] + for s in moduleSymbolStubs(ast.program, FileIndex modPos): + if ownsRuntimeRoutine(s, modPos): + names.add globalName(s, g.config) + writeLoweredArtifact(artifact, names) + proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = ## Per-module backend codegen (`--icBackendStage:cg --icBackendModule:`): ## generate C for the single module named by `icBackendModule` and write only @@ -365,6 +578,7 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = "per-module codegen: module not found for suffix: " & g.config.icBackendModule) return + applyLoweredBodies(g, modules, precompSys, target) generateCodeForModule(g, target) let bl = BModuleList(g.backend) # The main module also owns the whole-program method dispatchers + NimMain. @@ -502,7 +716,10 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) = proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = ## Main entry point for NIF-based C code generation. ## Traverses the module dependency graph and generates C code. - if g.config.icBackendStage == "cg": + if g.config.icBackendStage == "lower": + generateLowerStage(g, mainFileIdx) + return + elif g.config.icBackendStage == "cg": generateCgStage(g, mainFileIdx) return elif g.config.icBackendStage == "merge": @@ -516,4 +733,4 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = return else: rawMessage(g.config, errGenerated, - "the per-module NIF backend requires --icBackendStage:cg|merge|emit|link") + "the per-module NIF backend requires --icBackendStage:lower|cg|merge|emit|link") diff --git a/compiler/options.nim b/compiler/options.nim index 48b5ffbc93..3d62b006b3 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -29,7 +29,7 @@ const nimEnableCovariance* = defined(nimEnableCovariance) - icFormatVersion* = "14" + icFormatVersion* = "16" ## Version of the IC cache format (the sem-NIF module layout written by ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever ## that layout changes: `commandIc` wipes a nimcache whose `ic.version` @@ -786,6 +786,13 @@ template quitOrRaise*(conf: ConfigRef, msg = "") = else: quit(msg) # quits with QuitFailure +proc icLoweredBodies*(conf: ConfigRef): bool {.inline.} = + ## Whether the `nim ic` backend uses the EAGER per-module `lower` stage + ## (transformBody serialized to `.t.nif`, cg reuses it) instead of the lazy + ## Stage-0 path (cg re-derives every transformed body). This is now the + ## DEFAULT; `-d:icNoLowerBodies` opts back into the lazy path for A/B testing. + not isDefined(conf, "icNoLowerBodies") + proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.cmd in cmdDocLike + {cmdIdeTools} proc usesWriteBarrier*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc proc usesSso*(conf: ConfigRef): bool {.inline.} = conf.selectedStrings == stringSso