diff --git a/compiler/ast.nim b/compiler/ast.nim index 0d333d91f3..0c284a19f5 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -831,6 +831,10 @@ proc newSymNode*(sym: PSym): PNode = result = newNode(nkSym) result.sym = sym result.typField = sym.typ + if result.typField == nil and nifcBackendActive: + # See the two-arg overload in astdef: in the NIF backend cg stage a sym node + # built from a not-yet-typed stub must track the symbol's type lazily. + result.flags.incl nfLazyType result.info = sym.info proc newOpenSym*(n: PNode): PNode {.inline.} = diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index f1f5c787a2..3503040f9b 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -26,6 +26,13 @@ import typekeys import ic / [enum2nif] const SysModuleSuffix* = "@sys" +const BackendLocalMarker* = "@bk" + ## Suffix marker for a PROCESS-LOCAL backend-minted entity (a closure `:env` + ## type/obj/field/hidden-param minted while the VM compiles a routine body to + ## run a macro). Such entities have no stable cross-process identity, so each + ## module that references one emits its OWN module-local def named + ## `…@bk` and the loader homes it to the reading module with + ## a `backendItemId` (disjoint from real ids). See transf.transformBody. ## Reserved module-suffix sentinel for module-less magic singleton types — the ## `nil` type is created via `newSysType` with the graph idgen, whose `module` ## can be `-1` (e.g. during VM const-eval before a real module is current), so @@ -204,6 +211,13 @@ type # can keep mutating its still-live query targets writtenPackages: HashSet[string] depSuffixes: HashSet[string] # module suffixes already emitted as `(import ...)` deps + emittedBackendTypes: HashSet[int32] # backend-local type items already def'd this module + emittedBackendSyms: HashSet[int32] # backend-local sym items already def'd this module + lowering: bool # serializing the `lower` stage's whole-module `.t.nif` + emittedFieldSyms: HashSet[ItemId] # lowering: derived env-field syms already def'd + inTypeReclist: int # >0 while writing a type's OWN reclist: fields must be SELF-CONTAINED + # defs (the type can be seek-loaded in isolation), not entry-deduped uses + proc isLocalSym(sym: PSym): bool {.inline.} = ## Every symbol is emitted as a *global* (module-suffixed) name so that its @@ -232,7 +246,17 @@ const proc toNifSymName(w: var Writer; sym: PSym): string = ## Generate NIF name for a symbol: local names are `ident.disamb`, ## global names are `ident.disamb.moduleSuffix` - assert not sym.itemId.isBackendMinted + if sym.itemId.isBackendMinted: + # Process-local backend sym (closure env field / hidden `:env` param minted + # during a VM transform): re-home to the current module with the `@bk` + # marker so each referencing module self-contains it. See transformBody. + result = sym.name.s + result.add '.' + result.addInt sym.disamb + result.add '.' + result.add modname(w.currentModule, w.infos.config) + result.add BackendLocalMarker + return result = sym.name.s if sym.kindImpl == skPackage: result.add PkgMarker @@ -254,6 +278,11 @@ proc globalName*(sym: PSym; config: ConfigRef): string = result.addInt sym.disamb result.add '.' result.add modname(sym.itemId.module, config) + # A loaded process-local backend sym keeps its `@bk` marker in the NIF name + # (the index/`c.syms` tables are keyed by it); mirror toNifSymName so name-based + # lookups via globalName don't miss (KeyError `:env.N.` without the marker). + if sym.itemId.isBackendMinted: + result.add BackendLocalMarker type ParsedSymName* = object @@ -327,9 +356,24 @@ proc writeLoc(w: var Writer; dest: var TokenBuf; loc: TLoc) = writeFlags(dest, loc.flags) # TLocFlags dest.addStrLit loc.snippet +proc nifTypeName(w: Writer; typ: PType): string = + ## NIF name of a type as written by THIS module. A process-local backend env + ## type is re-homed to the current module with the `@bk` marker (see + ## BackendLocalMarker); everything else uses the canonical `typeToNifSym`. + if typ.uniqueId.isBackendMinted: + result = "`t" + result.addInt ord(typ.kind) + result.add '.' + result.addInt typ.uniqueId.item + result.add '.' + result.add modname(w.currentModule, w.infos.config) + result.add BackendLocalMarker + else: + result = typeToNifSym(typ, w.infos.config) + proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = dest.buildTree tdefTag: - dest.addSymDef pool.syms.getOrIncl(typeToNifSym(typ, w.infos.config)), NoLineInfo + dest.addSymDef pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo dest.addDotToken # always private for the index generator #dest.addIdent toNifTag(typ.kind) @@ -352,7 +396,13 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = writeType(w, dest, typ.typeInstImpl) #if typ.kind in {tyProc, tyIterator} and typ.nImpl != nil and typ.nImpl.kind != nkFormalParams: + # The reclist holds this type's OWN fields. A type can be force-loaded by + # name in isolation (cg seeks the `.t.nif`/`.s.nif` index entry), so its + # fields must be DEFS here, not entry-deduped SymUses whose def lives + # elsewhere in the `(lowered)` entry and is never read by the seek. + inc w.inTypeReclist writeNode(w, dest, typ.nImpl) + dec w.inTypeReclist writeSym(w, dest, typ.ownerFieldImpl) writeSym(w, dest, typ.symImpl) @@ -367,6 +417,15 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = if typ == nil: dest.addDotToken() + elif typ.uniqueId.isBackendMinted: + # Process-local closure env (see transf.transformBody): emit a MODULE-LOCAL + # `@bk` def the first time it is reached in this module, reference it after. + # Per-Writer dedup (NOT the shared `state`), since every referencing module + # must emit its own copy. + if not w.emittedBackendTypes.containsOrIncl(typ.uniqueId.item): + writeTypeDef(w, dest, typ) + else: + dest.addSymUse pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo elif typ.uniqueId.module == w.currentModule and typ.state == Complete: # Ownership for serialization is decided by `uniqueId`, not `itemId`: the NIF # name (`typeToNifSym`) and the loader (`createTypeStub`) both key off @@ -379,7 +438,7 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = if w.infos.config.ideActive: w.writtenTypes.add typ writeTypeDef(w, dest, typ) else: - dest.addSymUse pool.syms.getOrIncl(typeToNifSym(typ, w.infos.config)), NoLineInfo + dest.addSymUse pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo proc writeBool(dest: var TokenBuf; b: bool) = dest.buildTree (if b: "true" else: "false"): @@ -481,6 +540,18 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeLoc w, dest, sym.locImpl writeNode(w, dest, sym.constraintImpl) writeSym(w, dest, sym.instantiatedFromImpl) + # The TRANSFORMED body (ic_ideas.md 2-way body): a routine run at compile time + # (macro / VM transform / `static`) already has its lowered body — closure + # `:env` and all — computed during sem; serialize it so the backend reuses it + # instead of re-deriving (the divergence behind the t17.275 env class). An + # empty `.` here means "same as the semchecked body OR to be found in the + # `.t.nif`" (the `lower` stage fills that gap). Non-routines / not-yet- + # transformed routines write the empty marker. (`transformedBodyImpl` only + # exists in the routine branch of the `TSym` variant.) + if sym.kindImpl in routineKinds: + writeNode(w, dest, sym.transformedBodyImpl) + else: + dest.addDotToken dest.addParRi @@ -501,9 +572,31 @@ proc shouldWriteSymDef(w: var Writer; sym: PSym): bool {.inline.} = return true # Normal case for global symbols return false +proc isLoweredPerEntryField(w: Writer; sym: PSym): bool {.inline.} = + ## In the `lower` stage every entry (each `(lowered)` body AND each `@bk` type + ## def, which carries its fields inline) is loaded independently, so it must be + ## SELF-CONTAINED. A derived closure-env FIELD is module-homed (its id derives + ## from the captured local, NOT `@bk` — see itemids.derivedFieldId) so + ## `shouldWriteSymDef` would seal it after the first entry and later entries + ## would reference a def that their indexed copy does not contain. Re-emit it + ## as a full def per entry, deduped within the entry via `emittedFieldSyms`. + w.lowering and sym.kindImpl == skField and not sym.itemId.isBackendMinted + proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = if sym == nil: dest.addDotToken() + elif isLoweredPerEntryField(w, sym): + if not w.emittedFieldSyms.containsOrIncl(sym.itemId): + writeSymDef(w, dest, sym) + else: + dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo + elif sym.itemId.isBackendMinted: + # Process-local backend sym (closure env field / hidden `:env` param): emit a + # MODULE-LOCAL `@bk` def the first time, reference it after. Per-Writer dedup. + if not w.emittedBackendSyms.containsOrIncl(sym.itemId.item): + writeSymDef(w, dest, sym) + else: + dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo elif shouldWriteSymDef(w, sym): sym.state = Sealed if w.infos.config.ideActive: w.writtenSyms.add sym @@ -529,8 +622,23 @@ proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = var nodeTyp = n.typField if nodeTyp == nil and nfLazyType in n.flags: nodeTyp = sym.typImpl - if shouldWriteSymDef(w, sym): - sym.state = Sealed + # Backend-minted syms (process-local closure `:env` param/fields) are emitted + # as MODULE-LOCAL `@bk` defs the first time reached this module (per-Writer + # dedup shared with `writeSym`), regardless of module: their itemId.module is + # the systemModule of `vmTransfIdgen`, so `shouldWriteSymDef` (which gates on + # currentModule) would otherwise only ever emit a SymUse → dangling def. + let perEntryField = isLoweredPerEntryField(w, sym) + # A field reached while writing its own type's reclist MUST be a self-contained + # def: the type can be seek-loaded by name in isolation, so a deduped SymUse + # (whose def lives elsewhere in the entry) would resolve to nil. + let reclistField = w.lowering and w.inTypeReclist > 0 and sym.kindImpl == skField + let wantDef = + if reclistField: true + elif sym.itemId.isBackendMinted: not w.emittedBackendSyms.containsOrIncl(sym.itemId.item) + elif perEntryField: not w.emittedFieldSyms.containsOrIncl(sym.itemId) + else: shouldWriteSymDef(w, sym) + if wantDef: + if not sym.itemId.isBackendMinted and not perEntryField and not reclistField: sym.state = Sealed if w.infos.config.ideActive: w.writtenSyms.add sym if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): @@ -644,6 +752,7 @@ var importTag = registerTag("import") var implTag = registerTag("implementation") var reexpModTag = registerTag("reexpmod") var offerTag = registerTag("offer") +var typeOfferTag = registerTag("toffer") var modulesrcTag = registerTag("modulesrc") proc registerNifAstTags*() = @@ -676,6 +785,7 @@ proc registerNifAstTags*() = implTag = registerTag("implementation") reexpModTag = registerTag("reexpmod") offerTag = registerTag("offer") + typeOfferTag = registerTag("toffer") modulesrcTag = registerTag("modulesrc") proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = @@ -1293,6 +1403,7 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; genericOffers: seq[tuple[generic, inst: PSym; concreteTypes: seq[PType]; genericParamsCount: int]] = @[]; + typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[]; resolvedImportDeps: seq[FileIndex] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var content = createTokenBuf(300) @@ -1370,6 +1481,47 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; w.deps.addStrLit toFullPath(config, FileIndex(thisModule)) w.deps.addParRi + # Generic TYPE-instance OFFERS: the `tyGenericInst` types this module created + # (e.g. `HashArray[8192, Gwei]`). Non-IC keeps ONE such instance in the global + # `typeInstCache`, so a structural bound computed at the first instantiation + # site (e.g. an `array[…]` bound that depends on a `mixin`/`compiles()` whose + # resolution differs by import scope) is frozen and reused everywhere. A + # separate `nim m` process never repopulates `typeInstCache` from NIFs, so it + # re-instantiates in its own scope and can compute a DIFFERENT bound (the SSZ + # `dataPerChunk` divergence). The loader rebuilds `g.typeInstCache` from these + # so `semtypinst.searchInstTypes` hits and reuses the baked instance. + # Layout: (toffer ). + for off in typeOffers: + # Carry the generic body sym and the instance type as STRING LITERALS, not + # SymUse tokens: `addSymUse` rewrites a same-module reference into the NIF + # "local form" (suffix stripped, resolved by the content loader against the + # module being read), but this offer lives in the `deps` header and is read + # by a CONSUMER with no such module context. The full names round-trip + # verbatim as strings and `createTypeStub`/`resolveHookSym` resolve them + # directly (cf. `loadImport`, which carries module suffixes the same way). + w.deps.addParLe typeOfferTag, NoLineInfo + w.deps.addStrLit w.toNifSymName(off.generic) + w.deps.addStrLit typeToNifSym(off.inst, w.infos.config) + w.deps.addParRi + + # OWNER MUST EMIT: a type reachable only through an offered instance — the + # `concreteTypes` of an offered proc instance (e.g. chronicles `writeValue[T]`, + # where `T` is this module's own object type) or an offered generic type + # instance — may never be reached by the normal top-level serialization above. + # If this module OWNS such a type, force-emit its typedef so that a consumer + # which reuses the offer can resolve the cross-module SymUse to it. Without this + # the consumer writes `t..` and the loader asserts + # `symbol has no offset`. `writeType` emits the def (and recurses into owned + # sons) only for an own, still-Complete type; an already-Sealed one is skipped. + for off in genericOffers: + for ct in off.concreteTypes: + if ct != nil and ct.uniqueId.module == w.currentModule and ct.state == Complete: + writeType(w, bottom, ct) + for off in typeOffers: + if off.inst != nil and off.inst.uniqueId.module == w.currentModule and + off.inst.state == Complete: + writeType(w, bottom, off.inst) + # the implTag is used to tell the loader that the # bottom of the file is the implementation of the module: content.addParLe implTag, NoLineInfo @@ -1378,7 +1530,7 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; content.addParRi() let m = modname(w.currentModule, w.infos.config) - let nifFilename = AbsoluteFile(m).changeFileExt(".nif") + let nifFilename = AbsoluteFile(m).changeFileExt(".s.nif") let d = completeGeneratedFilePath(config, nifFilename).string var dest = createTokenBuf(600) @@ -1586,7 +1738,7 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): # but haven't had their NIF index loaded yet let hasEntry = c.mods.hasKey(result) if not hasEntry or AlwaysLoadInterface in flags: - let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string + let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.nif")).string if not fileExists(modFile): raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile & ". This can happen when loading a module from NIF that references another module " & @@ -1644,7 +1796,10 @@ proc tryCreateTypeStub(c: var DecodeContext; t: SymId): PType = let suffix = name.substr(i) if suffix == SysModuleSuffix: return reconstructSysType(c, name, k, itemVal) - let id = itemId(moduleId(c, suffix).int32, itemVal) + let isBk = suffix.endsWith(BackendLocalMarker) + let realSuffix = if isBk: suffix[0 ..< suffix.len - BackendLocalMarker.len] else: suffix + let modIdx = moduleId(c, realSuffix).int32 + let id = if isBk: backendItemId(modIdx, itemVal) else: itemId(modIdx, itemVal) let ii = addr c.mods[id.module.FileIndex].index let offs = ii[].getOrDefault(name) if offs.offset == 0: @@ -1671,9 +1826,13 @@ proc createTypeStub(c: var DecodeContext; t: SymId): PType = let suffix = name.substr(i) if suffix == SysModuleSuffix: return reconstructSysType(c, name, k, itemVal) - let id = itemId(moduleId(c, suffix).int32, itemVal) - let ii = addr c.mods[id.module.FileIndex].index - let offs = ii[].getOrDefault(name) + let isBk = suffix.endsWith(BackendLocalMarker) + let realSuffix = if isBk: suffix[0 ..< suffix.len - BackendLocalMarker.len] else: suffix + let modIdx = moduleId(c, realSuffix).int32 + let id = if isBk: backendItemId(modIdx, itemVal) else: itemId(modIdx, itemVal) + let modFi = id.module.FileIndex + let ii = addr c.mods[modFi].index + var offs = ii[].getOrDefault(name) if offs.offset == 0: raiseAssert "symbol has no offset: " & name result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) @@ -1759,10 +1918,16 @@ proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string; # Global symbol - look up in index for lazy loading result = c.syms.getOrDefault(symAsStr)[0] if result == nil: - let module = moduleId(c, sn.module) + # A process-local backend sym (closure env field / `:env` param) is named + # `…@bk`: home it to that module with a backendItemId so it + # stays disjoint from the loader's real per-module id space (see toNifSymName). + let isBk = sn.module.endsWith(BackendLocalMarker) + let realMod = if isBk: sn.module[0 ..< sn.module.len - BackendLocalMarker.len] + else: sn.module + let module = moduleId(c, realMod) let val = addr c.mods[module].symCounter inc val[] - let id = itemId(module.int32, val[]) + let id = if isBk: backendItemId(module.int32, val[]) else: itemId(module.int32, val[]) let offs = c.mods[module].index.getOrDefault(symAsStr) if offs.offset == 0: @@ -1843,7 +2008,12 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms raiseAssert "(td) expected" var scanCursor = n # copy cursor at start of type - let typesModule = parseSymName(pool.syms[n.firstSon.symId]).module + var typesModule = parseSymName(pool.syms[n.firstSon.symId]).module + if typesModule.endsWith(BackendLocalMarker): + # A backend-minted (`@bk`) type's name carries the marker in its module part; + # strip it so the nested-local pre-scan resolves the real module, not a + # nonexistent `@bk.nif`. + typesModule = typesModule[0 ..< typesModule.len - BackendLocalMarker.len] extractLocalSymsFromTree(c, scanCursor, typesModule, localSyms) inc n # move past (td @@ -1882,8 +2052,18 @@ proc loadType*(c: var DecodeContext; t: PType) = if t.state != Partial: return t.state = c.loadedState var buf = createTokenBuf(30) - let typeName = typeToNifSym(t, c.infos.config) - var n = cursorFromIndexEntry(c, t.itemId.module.FileIndex, c.types[typeName][1], buf) + # A backend-minted (`@bk`) closure-env type produced by the `lower` stage lives + # ONLY in the `.t.nif` and is keyed by its `@bk` name (see nifTypeName), not the + # canonical `typeToNifSym` (which asserts non-`@bk`). Reconstruct that name so a + # Partial `@bk` stub that escaped the inline pre-scan can still be force-loaded. + let typeName = + if t.uniqueId.isBackendMinted: + "`t" & $ord(t.kind) & "." & $t.uniqueId.item & "." & + modname(t.itemId.module, c.infos.config) & BackendLocalMarker + else: + typeToNifSym(t, c.infos.config) + let modFi = t.itemId.module.FileIndex + var n = cursorFromIndexEntry(c, modFi, c.types[typeName][1], buf) var localSyms = initTable[string, PSym]() loadTypeFromCursor(c, n, t, localSyms) @@ -1968,6 +2148,15 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: loadLoc c, n, s.locImpl s.constraintImpl = loadNode(c, n, thisModule, localSyms) s.instantiatedFromImpl = loadSymStub(c, n, thisModule, localSyms) + # The TRANSFORMED body slot (see writeSymDef). Reconstruct it ONLY in the + # backend (`cmdNifC`), where `transformBody` short-circuits on it; during + # frontend sem (`cmdM`) skip the tokens — a dependent never needs a foreign + # routine's lowered body, and reconstructing one must not perturb effect/ + # exception inference (the "never change frontend node-typing for IC" rule). + if c.infos.config.cmd == cmdNifC and s.kindImpl in routineKinds: + s.transformedBodyImpl = loadNode(c, n, thisModule, localSyms) + else: + skip n skipParRi n proc loadSym*(c: var DecodeContext; s: PSym) = @@ -2000,6 +2189,21 @@ proc loadSym*(c: var DecodeContext; s: PSym) = if docId != 0'u32 and s.astImpl != nil and nodeCommentWriter != nil: nodeCommentWriter(s.astImpl, pool.strings[StrId(docId)]) +proc sealLoadedRoutines*(c: var DecodeContext) = + ## Before `writeLoweredModule` re-serializes the lowered module, seal ONLY the + ## module's ROUTINE syms. A `.t.nif` written by `writeLoweredModule` is the + ## SOLE source the `cg` stage loads (there is no `.s.nif` fallback for its + ## bodies), so every type, global, param and local must still emit a REAL def + ## in it — only cross-routine references may be `SymUse`s (each routine's def is + ## emitted once, at module scope, by the explicit stub loop). Types/globals stay + ## `Complete` so `writeType`/`writeGlobals` emit them; routines become `Sealed` + ## so a body referencing another routine writes a `SymUse` resolved through the + ## module index. + for _, v in c.syms: + if v[0] != nil and v[0].state == Complete and v[0].kindImpl in routineKinds: + v[0].state = Sealed + +proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info) @@ -2073,9 +2277,46 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; loadSymFromCursor(c, sym, n, thisModule, localSyms) sym.state = c.loadedState # mark as fully loaded result = newSymNode(sym, info) - else: + elif sn.module.endsWith(BackendLocalMarker): + # A backend-minted (`@bk`) def lives ONLY inline in this `.t.nif` body + # (not in any module index): create/find its cached stub and FILL it + # from the sdef instead of skipping (which would leave the skModule/ + # Partial stub `loadSymStub` made unresolved). sym = c.loadSymStub(name.symId, thisModule, localSyms) - skip n # skip the entire sdef for indexed symbols + if sym.state == Partial: + sym.state = c.loadedState + inc n # skip `sd` tag + loadSymFromCursor(c, sym, n, thisModule, localSyms) + else: + skip n + result = newSymNode(sym, info) + result.flags.incl nfLazyType + else: + # A module-homed inline sdef. Normally its def lives in that module's + # index and is loaded lazily, so we skip the inline copy. BUT a + # transform-created closure-env FIELD (`x0.0.clo`) is module-homed yet + # lives ONLY inline in this `.t.nif` reclist — it has no index entry. + # Skipping it leaves a nil-typed `skModule` fallback stub (from + # loadSymStub's "no offset" path) and codegen of the env struct then + # dereferences a nil field type. Detect the unindexed case and FILL + # the sym from the inline def instead. + let m = moduleId(c, sn.module) + let indexed = c.mods[m].index.hasKey(symName) + if indexed: + sym = c.loadSymStub(name.symId, thisModule, localSyms) + skip n # skip the entire sdef for indexed symbols + else: + sym = c.syms.getOrDefault(symName)[0] + if sym == nil: + let val = addr c.mods[m].symCounter + inc val[] + sym = PSym(itemId: itemId(m.int32, val[]), kindImpl: skStub, + name: c.cache.getIdent(sn.name), disamb: sn.count.int32, + state: Partial) + c.syms[symName] = (sym, NifIndexEntry()) + sym.state = c.loadedState + inc n # skip `sd` tag + loadSymFromCursor(c, sym, n, thisModule, localSyms) result = newSymNode(sym, info) result.flags.incl nfLazyType of typeDefTagName: @@ -2166,11 +2407,14 @@ proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; if result == nil: let symAsStr = nifName let sn = parseSymName(symAsStr) - let symModule = moduleId(c, if sn.module.len > 0: sn.module else: thisModule) + let rawMod = if sn.module.len > 0: sn.module else: thisModule + let isBk = rawMod.endsWith(BackendLocalMarker) + let realMod = if isBk: rawMod[0 ..< rawMod.len - BackendLocalMarker.len] else: rawMod + let symModule = moduleId(c, realMod) let val = addr c.mods[symModule].symCounter inc val[] - let id = itemId(symModule.int32, val[]) + let id = if isBk: backendItemId(symModule.int32, val[]) else: itemId(symModule.int32, val[]) let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, entry) @@ -2233,7 +2477,15 @@ proc moduleSymbolStubs*(c: var DecodeContext; module: FileIndex): seq[PSym] = proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) - result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string + # The `cg`/`emit` backend stages load the lowered whole-module NIF (transformed + # bodies + lifted sigs baked in); the `lower` stage and the frontend (`cmdM`) + # read the semchecked `.s.nif`. + if conf.cmd == cmdNifC and + (conf.icBackendStage == "cg" or conf.icBackendStage == "emit"): + let t = toGeneratedFile(conf, AbsoluteFile(suffix), ".t.nif").string + if fileExists(t): + return t + result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.nif").string proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym = result = c.syms.getOrDefault(symAsStr)[0] @@ -2243,7 +2495,10 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo let sn = parseSymName(symAsStr) if sn.module.len == 0: return nil # Local symbols shouldn't be hooks - let module = moduleId(c, sn.module) + let isBk = sn.module.endsWith(BackendLocalMarker) + let realMod = if isBk: sn.module[0 ..< sn.module.len - BackendLocalMarker.len] + else: sn.module + let module = moduleId(c, realMod) # Look up the symbol in the module's index # Try both formats: with module suffix (e.g., "foo.0.modulename") and without (e.g., "foo.0.") # NIF spec allows local symbols to be stored without module suffix @@ -2259,7 +2514,7 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo # Create a stub symbol let val = addr c.mods[module].symCounter inc val[] - let id = itemId(int32(module), val[]) + let id = if isBk: backendItemId(int32(module), val[]) else: itemId(int32(module), val[]) result = PSym(itemId: id, kindImpl: skProc, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, offs) @@ -2356,6 +2611,11 @@ type ## generic instances this module created; modulegraphs.nim rebuilds ## `procInstCache` from them so a consumer reuses the instance instead of ## re-instantiating it in its own (operator-blind) module scope. + typeOffers*: seq[tuple[generic: PSym; inst: PType]] + ## generic TYPE instances this module created; modulegraphs.nim rebuilds + ## `typeInstCache` from them so a consumer reuses the baked instance + ## (e.g. a `mixin`/`compiles()`-dependent array bound) instead of + ## re-instantiating it with a different bound in its own scope. includes*: seq[string] # resolved full paths of files this module `include`s; # replayed into `inclToMod` by modulegraphs.nim so that # nimsuggest can map a query in an include file back to @@ -2625,6 +2885,35 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; t = next(s) if ok and genSym != nil and instSym != nil: result.genericOffers.add (genSym, instSym, cts, paramsCount) + elif t.tagId == typeOfferTag: + # (toffer "" "") — see the writer. The two + # full names arrive as string literals; intern them and resolve to a + # PSym/PType, then FULLY load the instance (its array bounds/fields) so + # `searchInstTypes` can match its params (a Partial stub has empty kids). + # Best-effort: a type that fails to resolve drops the offer. + t = next(s) # skip (toffer + var genName, instName = "" + var idx = 0 + while t.kind != ParRi and t.kind != EofToken: + if t.kind == StringLit: + if idx == 0: genName = pool.strings[t.litId] + elif idx == 1: instName = pool.strings[t.litId] + inc idx + t = next(s) + if t.kind != ParRi: + raiseAssert "expected ParRi in toffer entry of module " & suffix + t = next(s) + if genName.len > 0 and instName.len > 0: + # Resolving/loading these entities lazily reads from the SAME stream we + # are iterating, moving its cursor — save/restore around it (cf. the + # export-list handling above). + let saved = offset(s.r) + let genSym = resolveHookSym(c, pool.syms.getOrIncl(genName)) + let inst = tryCreateTypeStub(c, pool.syms.getOrIncl(instName)) + if genSym != nil and inst != nil: + loadType(c, inst) + result.typeOffers.add (genSym, inst) + s.r.jumpTo(saved) elif t.tagId == modulesrcTag: # self-identification record for the standalone include-graph scanner; # not needed by the lazy loader, just skip past it. @@ -2675,6 +2964,112 @@ proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: va let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f)) result = loadNifModule(c, suffix, interf, interfHidden, flags) +proc writeLoweredModule*(c: var DecodeContext; config: ConfigRef; + precomp: PrecompiledModule; + hooks: openArray[LogEntry]; outfile: string) = + ## Re-serialize a backend-loaded module as a FULL module NIF (`.t.nif`) whose + ## routine `(sd)` entries carry their TRANSFORMED bodies (the `lower` stage set + ## them, recursively lifting nested closures — including the async state-machine + ## procs whose inner closure the per-`(lowered)`-entry path failed to cross) and + ## whose lambda-lift-minted entities (closure-env types/syms, lifted nested + ## procs) are real, indexed defs. The `cg` stage then loads it through the + ## normal module loader (`moduleFromNifFile`), so a transformed body arrives via + ## `loadSymFromCursor`'s Step-A 2-way-body slot WITH the lifted signature — no + ## `(lowered)` side-car, no `:envP` re-weld. This realizes `ic_ideas.md`'s eager + ## two-way body whole-module. + let thisModule = precomp.module.positionImpl.int32 + # Routines → Sealed (cross-routine refs become SymUse, defs emitted once below); + # types/globals/params/locals stay Complete and emit real defs (the `.t.nif` is + # the sole source the cg stage reads — no `.s.nif` fallback for them). + sealLoadedRoutines(c) + var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) + w.inProc = 1 + w.lowering = true + var content = createTokenBuf(300) + let rootInfo = trLineInfo(w, precomp.topLevel.info) + createStmtList(content, rootInfo) + + # This module's ops (hooks/converters/methods/pure-enums) loaded from `.s.nif`, + # plus the type-bound ops the lower transform just lifted (closure-env + # `=destroy` etc., which have no `.s.nif` entry). + for op in precomp.logOps: + if op.module == thisModule.int: + writeOp(w, content, op) + for op in hooks: + writeOp(w, content, op) + + var bottom = createTokenBuf(300) + # Imperative init code + global let/var/const sections + replay actions — all + # that a backend-loaded `topLevel` carries (routines are lazy index sdefs, not + # here). Emits + seals the module's globals. + w.writeToplevelNode content, bottom, precomp.topLevel + + # Routine DEFS with transformed bodies, sourced from the index. + for s in moduleSymbolStubs(c, FileIndex thisModule): + if s.kindImpl in routineKinds and s.itemId.module == thisModule: + writeSymDef(w, bottom, s) + + # Lifted hook ROUTINES (`@bk`, NEW in the lower stage — no `.s.nif` sdef, so + # absent from `moduleSymbolStubs`): emit each as a full def (sig + transformed + # body) so `injectDestructorCalls` in cg resolves the loaded env's `=destroy`. + var emittedHooks = initHashSet[int32]() + for op in hooks: + if op.sym != nil and op.sym.kindImpl in routineKinds and + not emittedHooks.containsOrIncl(op.sym.itemId.item): + writeSymDef(w, bottom, op.sym) + + # deps / reexports / offers — mirror writeNifModule so the cg backend closure + # walk, interface re-export and generic-instance reuse all work off `.t.nif`. + for dep in precomp.deps: + if not w.depSuffixes.containsOrIncl(dep.string): + w.deps.addParLe importTag, NoLineInfo + w.deps.addDotToken + w.deps.addDotToken + w.deps.addStrLit dep.string + w.deps.addParRi + for (mname, msuffix) in precomp.reexportedModules: + w.deps.addParLe reexpModTag, NoLineInfo + w.deps.addStrLit mname + w.deps.addStrLit msuffix + w.deps.addParRi + for off in precomp.genericOffers: + w.deps.addParLe offerTag, NoLineInfo + w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.generic)), NoLineInfo + w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.inst)), NoLineInfo + w.deps.addIntLit off.genericParamsCount + for ct in off.concreteTypes: + w.deps.addSymUse pool.syms.getOrIncl(typeToNifSym(ct, w.infos.config)), NoLineInfo + w.deps.addParRi + for off in precomp.typeOffers: + w.deps.addParLe typeOfferTag, NoLineInfo + w.deps.addStrLit w.toNifSymName(off.generic) + w.deps.addStrLit typeToNifSym(off.inst, w.infos.config) + w.deps.addParRi + # OWNER MUST EMIT offered types this module owns (see writeNifModule). + for off in precomp.genericOffers: + for ct in off.concreteTypes: + if ct != nil and ct.uniqueId.module == w.currentModule and ct.state == Complete: + writeType(w, bottom, ct) + for off in precomp.typeOffers: + if off.inst != nil and off.inst.uniqueId.module == w.currentModule and + off.inst.state == Complete: + writeType(w, bottom, off.inst) + + # Assemble exactly as writeNifModule: (stmts . . + # (implementation) ). + content.addParLe implTag, NoLineInfo + content.addParRi() + content.add bottom + content.addParRi() + + var dest = createTokenBuf(600) + createStmtList(dest, rootInfo) + dest.add w.deps + for i in 3 ..< content.len-1: + dest.add content[i] + dest.addParRi() + writeFile(dest, outfile) + when isMainModule: import std / syncio let obj = parseSymName("a.123.sys") diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 334bbdc945..63de1814fb 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -17,6 +17,15 @@ when defined(nimPreviewSlimSystem): export int128 +var nifcBackendActive* = false + ## Set only while the per-module NIF backend codegen stage runs + ## (`nifbackend.generateCgStage`, `cmd == cmdNifC`). It gates `newSymNode`'s + ## lazy-type marking so it applies ONLY in the backend — where syms are loaded + ## from NIF and a cg-stage transform can build a sym node from a not-yet-typed + ## stub — and never during frontend sem, where the same marking would perturb + ## effect/exception inference (it diverges from a non-IC build, e.g. + ## `times.toDateTimeByWeek` gaining a spurious unlisted `Exception`). + import nodekinds export nodekinds @@ -970,6 +979,14 @@ proc newSymNode*(sym: PSym, info: TLineInfo): PNode = result = newNode(nkSym) result.sym = sym result.typField = sym.typImpl + if result.typField == nil and nifcBackendActive: + # In the per-module NIF backend cg stage a transform (chronos async + # closure-iterator lowering) builds `result = …` sym nodes from a not-yet-typed + # NIF stub; snapshotting the nil here would leave the node permanently typeless + # and the backend later reads `t.flags` off it and SIGSEGVs (injectdestructors + # hasDestructor). Mark it lazy so `typ` re-reads `sym.typ` once resolved. Gated + # on `nifcBackendActive` so frontend sem is untouched (see the flag's doc). + result.flags.incl nfLazyType result.info = info proc newStrNode*(kind: TNodeKind, strVal: string): PNode = diff --git a/compiler/ccgthreadvars.nim b/compiler/ccgthreadvars.nim index 41a1ce69ba..73fe96306e 100644 --- a/compiler/ccgthreadvars.nim +++ b/compiler/ccgthreadvars.nim @@ -39,11 +39,30 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) = if isExtern: Extern elif lfExportLib in s.loc.flags: ExportLibVar else: Private - m.s[cfsVars].addVar(m, s, - name = s.loc.snippet, - typ = getTypeDesc(m, s.loc.t), - kind = Threadvar, - visibility = vis) + if m.config.cmd == cmdNifC and vis == Private and not isExtern: + # A `{.threadvar.}`/`{.global.}` thread-local declared inside a routine is + # emitted by every module that emit-everywhere's its enclosing routine + # (e.g. libp2p's `var keys {.global.}: HashSet`), so its content-addressed + # name collides at link. Same fix as a plain global (genGlobalVarDecl): + # `extern` declaration + a droppable `'d'` definition unit the merge stage + # assigns one owner. The thread-local storage class rides on both. + let cname = stripCnifMarks(s.loc.snippet) + let td = getTypeDesc(m, s.loc.t) + # `extern` declaration via the full `addVar` overload — it knows the + # thread-local storage class (`NIM_THREADVAR`); the simple `addVar`'s + # `addVarHeader` does not implement `Threadvar`. + m.s[cfsVars].addVar(m, s, name = s.loc.snippet, typ = td, + kind = Threadvar, visibility = Extern) + m.s[cfsVars].add(cnifDefDirective(cname, "d", icNifName(m, s))) + m.s[cfsVars].addVar(m, s, + name = s.loc.snippet, typ = td, kind = Threadvar, visibility = vis) + m.s[cfsVars].add(cnifEndDefs()) + else: + m.s[cfsVars].addVar(m, s, + name = s.loc.snippet, + typ = getTypeDesc(m, s.loc.t), + kind = Threadvar, + visibility = vis) proc generateThreadLocalStorage(m: BModule) = if m.g.nimtv.buf.len != 0 and (usesThreadVars in m.flags or sfMainModule in m.module.flags): diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index eb82c854c7..2fc37f6562 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -108,7 +108,14 @@ proc fillBackendName(m: BModule; s: PSym) = var result: Rope if s.kind in routineKinds and {optCDebug, optItaniumMangle} * m.g.config.globalOptions == {optCDebug, optItaniumMangle} and m.g.config.symbolFiles == disabledSf: - result = mangleProc(m, s, false).rope + # Under the per-module IC backend the bare-name uniqueness probe + # (`m.g.mangledPrcs`) only sees the routines of the CURRENT module, so the + # clean-vs-`makeUnique` decision is made independently per process: a + # method base mangles clean at its owner but loses the in-module race to + # its same-signature dispatcher elsewhere (clean `speak` defined twice -> + # "multiple definition"; demanders call `speak_u` that nobody defines). + # Force the stable, disamb-based unique name so every process agrees. + result = mangleProc(m, s, makeUnique = m.config.cmd == cmdNifC).rope else: let shared = sharedInstanceCName(m, s) if shared.len > 0: @@ -1408,10 +1415,24 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; m.hcrCreateTypeInfosProc.addCast(typ = ptrType(CPointer)): m.hcrCreateTypeInfosProc.add(cAddr(name)) else: - m.s[cfsStrData].addDeclWithVisibility(Private): - m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType") if m.config.cmd == cmdNifC: + # Emit-everywhere (see genTypeInfoV1's perModuleCg gate): every demanding + # `cg` process emits this type info's tentative definition. Declare it + # `extern` first (the data analogue of a proc prototype) so a TU whose copy + # the merge stage drops still has a valid declaration; wrap the definition + # as a droppable `'d'` unit the merge stage assigns to a single owner so + # exactly one external-linkage tentative definition survives (preserving + # the RTTI pointer identity refc relies on). + m.s[cfsStrData].addDeclWithVisibility(Extern): + m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType") + m.s[cfsStrData].add(cnifDefDirective(name, "d", icNifName(m, origType))) + m.s[cfsStrData].addDeclWithVisibility(Private): + m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType") + m.s[cfsStrData].add(cnifEndDefs()) m.icDataDefs.add (name, icNifName(m, origType)) + else: + m.s[cfsStrData].addDeclWithVisibility(Private): + m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType") proc genTypeInfoAux(m: BModule; typ, origType: PType, name: Rope; info: TLineInfo) = @@ -1504,8 +1525,25 @@ proc genObjectFields(m: BModule; typ, origType: PType, n: PNode, expr: Rope; m.s[cfsTypeInit3].addFieldAssignment(expr, "name", makeCString(field.name.s)) m.s[cfsTypeInit3].addFieldAssignment(expr, "sons", cAddr(subscript(tmp, cIntValue(0)))) m.s[cfsTypeInit3].addFieldAssignment(expr, "len", L) - m.s[cfsData].addArrayVar(kind = Local, name = tmp, - elementType = ptrType("TNimNode"), len = toInt(L)+1) + if m.config.cmd == cmdNifC: + # The discriminator table has a content-addressed name + # (`NimDT__`) and is emitted by every module that demands + # this variant type's RTTI (emit-everywhere; RTTI has no single owner — + # emission is lazy and often skipped). Declare it `extern` + wrap the + # tentative definition as a droppable `'d'` unit so the merge stage keeps + # exactly one external-linkage definition (mirrors the `TNimType` var and + # consts); otherwise the identical name collides across modules at link. + m.s[cfsData].addDeclWithVisibility(Extern): + m.s[cfsData].addArrayVar(kind = Local, name = tmp, + elementType = ptrType("TNimNode"), len = toInt(L)+1) + m.s[cfsData].add(cnifDefDirective(tmp, "d", "")) + m.s[cfsData].addArrayVar(kind = Local, name = tmp, + elementType = ptrType("TNimNode"), len = toInt(L)+1) + m.s[cfsData].add(cnifEndDefs()) + m.icDataDefs.add (tmp, "") + else: + m.s[cfsData].addArrayVar(kind = Local, name = tmp, + elementType = ptrType("TNimNode"), len = toInt(L)+1) for i in 1..` scheme while a sibling that read the project's + # config.nims (`--debugger:native`) used Itanium, so the same symbol's + # definition and cross-module references would disagree at link. + if optCDebug in c.config.globalOptions: + result.add "--debugger:native" # the children compile each MODULE as their own project file, which makes # that module's package the "main package" and unfilters foreign-package # diagnostics — a vendored package's hintAsError/warningAsError promotions @@ -918,6 +926,36 @@ proc backendCFile(c: DepContext; node: Node): string = result = changeFileExt(completeCfilePath(c.config, mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string +proc computeLiveBackendNodes(c: DepContext): seq[bool] = + ## Which nodes the backend must code-generate: the closure reachable from the + ## program roots (main + `system` + `--import`ed modules) via the REAL, + ## post-sem import edges (`.s.deps`). + ## + ## The static `.deps` scan over-approximates: it cannot evaluate guards like + ## `when defined(windows)` or const-aliased ones (`when useWinVersion`, with + ## `const useWinVersion = defined(windows) or defined(nimdoc)`), so it keeps + ## the dead branch's import. e.g. on Linux `nativesockets`'s static deps list + ## `winlean`; the discovery fixpoint only ever *adds* edges, never prunes, so + ## `winlean` stays a node and got a full `lower`/`cg`/`emit`/link pipeline. + ## That is harmless for sem (an extra `nim m`) but fatal for codegen: + ## `winlean`'s `importc, header: "winsock2.h"` decls emit + ## `#include "winsock2.h"` into a C file that cannot compile off-Windows. + ## Sem's resolved import set (`.s.deps`) is the real program graph — the + ## non-IC compiler would never touch `winlean` here — so restrict the backend + ## to it. (`.s.deps` is the same data the discovery loop trusts; it is written + ## for every sem'd module, including grouped SCC members.) + result = newSeq[bool](c.nodes.len) + var stack: seq[int] = @[0] # main module + if c.systemNodeId >= 0: stack.add c.systemNodeId + for impId in c.implicitNodeIds: stack.add impId # every module imports these + while stack.len > 0: + let ni = stack.pop() + if ni < 0 or ni >= c.nodes.len or result[ni]: continue + result[ni] = true + for p in readSemDeps(c, c.nodes[ni].files[0]): + let idx = c.processedModules.getOrDefault(c.toPair(p).modname, -1) + if idx >= 0: stack.add idx + proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string = ## Per-module backend build file. One `nim_nifc` command template (the actual ## stage/module switches ride in each rule's `(args …)`), then the stages of @@ -941,9 +979,42 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string # Per-node output paths. var cnifFiles = newSeq[string](c.nodes.len) var cFiles = newSeq[string](c.nodes.len) + var tFiles = newSeq[string](c.nodes.len) + # The `lower` stage writes a PROPER module NIF the cg/emit stages load via + # `toNifFilename` (a `.s.nif` sibling), so its `.t.nif` lives at the suffix base + # (mirroring `semmedFile`), not next to the throwaway `.c`. for i, node in c.nodes: cFiles[i] = backendCFile(c, node) cnifFiles[i] = cFiles[i] & ".nif" + tFiles[i] = nimcache / node.files[0].modname & ".t.nif" + + # Only code-generate modules the real program actually reaches; statically + # over-approximated nodes (e.g. `winlean` on Linux) are sem'd but not emitted. + let live = computeLiveBackendNodes(c) + # Drop a pruned node's stale backend artifacts: the `merge` stage globs + # `*.c.nif` off disk (not the build-file inputs) and the `link` stage scans + # the loaded closure's `.c`s, so a leftover `.c.nif`/`.c` from a run before + # this module became unreachable (a prior over-approximated build, or an edit + # that removed its last real importer) would still be merged/compiled — + # reintroducing exactly the off-platform `#include` this prune avoids. + var prunedStale = false + for i in 0 ..< c.nodes.len: + if not live[i]: + # `fileExists` before remove so we only force a merge recompute (below) + # when an artifact was actually present — i.e. a build where this module + # WAS emitted, not the steady state where it never is. + if fileExists(cnifFiles[i]) or fileExists(cFiles[i]): prunedStale = true + removeFile(cnifFiles[i]) + removeFile(cFiles[i]) + # The merge decision is a pure function of the set of `.c.nif`s present; if we + # just removed an over-approximated module's artifacts, a decision computed + # while they were present is stale — it can name a now-absent module as a + # symbol's owner (`asyncdispatch` owning `NTIdomain` here), leaving that symbol + # undefined at link. nifmake will not re-fire `merge` on its own: dropping an + # input makes no remaining input newer than the output. Delete the decision so + # the (now missing) output forces a recompute against the live `.c.nif` set. + if prunedStale: + removeFile(mergeFile) var b = nifbuilder.open(result) defer: b.close() @@ -965,9 +1036,12 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string b.addStrLit a b.addTree "args" b.endTree() - b.addTree "input" - b.addIntLit 0 - b.endTree() + # The project file is a fixed command ARGUMENT, not a tracked input: backend + # stages read NIFs (resolved by suffix), never the `.nim` source, so its + # content cannot change any artifact. Passing it as `(input 0)` made its mtime + # an input to every rule, so editing the main module's source re-fired the + # whole backend. + b.addStrLit mainNif b.endTree() template inputStr(s: string) = @@ -979,21 +1053,50 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string b.addStrLit s b.endTree() - # cg: one rule per module. Inputs are the project (slot 0) and every semmed - # NIF (so the whole program loads and the rule is ordered after the frontend); - # the main module additionally depends on every other `.c.nif` (init metas). + # lower: one rule per module. Transforms (eventually) the routines the module + # OWNS once, in the owner's id space, into `.t.nif`, so the `cg` stage + # reads them instead of re-deriving (which makes a closure `:env`'s identity + # diverge across the parallel `cg` processes). Runs per module in parallel. + # + # Input is this module's OWN semmed NIF and nothing else. A module does NOT + # depend on its importers, so listing every semmed NIF (or even the import + # closure) was wrong: it made e.g. `strutils`'s rule depend on the `finish` + # that imports it. nifmake handles the indirect dependency for free — the + # frontend writes `.s.nif`s content-stably, so an interface change to a + # dependency re-sems (and re-emits the `.s.nif` of) every transitive importer; + # a module whose own `.s.nif` is unchanged genuinely needs no re-lowering. for i, node in c.nodes: + if not live[i]: continue + b.addTree "do" + b.addIdent "nim_nifc" + b.withTree "args": + b.addStrLit "--icBackendStage:lower" + b.addStrLit "--icBackendModule:" & node.files[0].modname + inputStr c.semmedFile(node.files[0]) + outputStr tFiles[i] + b.endTree() + + # cg: one rule per module. Input is this module's OWN `.t.nif`. cg DOES read + # its dependencies' `.t.nif`s at runtime (loadDepClosure), but ordering is + # guaranteed by nifmake's depth-barriered scheduler: every `lower` is depth 1 + # (its `.s.nif` is a leaf) and every `cg` is depth 2, so all lowering finishes + # before any cg starts — no need to list the closure for ordering. For + # invalidation, a dependency's change reaches this module through its own + # `.t.nif` (own `.s.nif` re-sem -> own `lower`); a foreign body this module + # emit-everywhere'd but does not own is dropped by `emit` regardless, so a + # stale copy here is harmless. The main module additionally depends on every + # other `.c.nif` (it reads their init/datInit metas to wire up NimMain). + for i, node in c.nodes: + if not live[i]: continue b.addTree "do" b.addIdent "nim_nifc" b.withTree "args": b.addStrLit "--icBackendStage:cg" b.addStrLit "--icBackendModule:" & node.files[0].modname - inputStr mainNif - for n2 in c.nodes: - inputStr c.semmedFile(n2.files[0]) + inputStr tFiles[i] if node.id == 0: for j in 0 ..< c.nodes.len: - if c.nodes[j].id != 0: + if c.nodes[j].id != 0 and live[j]: inputStr cnifFiles[j] outputStr cnifFiles[i] b.endTree() @@ -1003,19 +1106,24 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string b.addIdent "nim_nifc" b.withTree "args": b.addStrLit "--icBackendStage:merge" - inputStr mainNif - for cn in cnifFiles: inputStr cn + for i in 0 ..< c.nodes.len: + if live[i]: inputStr cnifFiles[i] outputStr mergeFile b.endTree() # emit: render each module's `.c` from its `.c.nif` + the merge decision. for i, node in c.nodes: + if not live[i]: continue b.addTree "do" b.addIdent "nim_nifc" b.withTree "args": b.addStrLit "--icBackendStage:emit" b.addStrLit "--icBackendModule:" & node.files[0].modname - inputStr mainNif + # Inputs: this module's OWN `.c.nif` and the global merge decision. emit also + # loads `.t.nif`s at runtime (getCFile/type resolution), but those are depth 1 + # and emit is past the merge barrier, so they always exist — no need to list + # them. (emit still re-fires for every module whenever `merge` rewrites the + # decision file; making that incremental is a separate concern.) inputStr cnifFiles[i] inputStr mergeFile outputStr cFiles[i] @@ -1026,8 +1134,8 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string b.addIdent "nim_nifc" b.withTree "args": b.addStrLit "--icBackendStage:link" - inputStr mainNif - for cf in cFiles: inputStr cf + for i in 0 ..< c.nodes.len: + if live[i]: inputStr cFiles[i] outputStr exeFile b.endTree() @@ -1150,8 +1258,14 @@ proc commandIc*(conf: ConfigRef) = # each DAG depth via execProcesses (defaults to all cores). Cold builds are # otherwise serial (one child at a time) and leave the machine idle. Opt out # with `-d:icNoParallel` (e.g. for readable, non-interleaved child output - # when debugging a build). - let parallel = if isDefined(conf, "icNoParallel"): "" else: " --parallel" + # when debugging a build), or cap the concurrency with `-d:icJobs:N` — an + # uncapped fan-out across many cores can exhaust RAM on a large project + # (each `nim m`/`cg` child holds its own module graph), which nifmake's own + # `-j:N` exists to bound. + let parallel = + if isDefined(conf, "icNoParallel"): "" + elif isDefined(conf, "icJobs"): " --parallel:" & conf.symbols["icJobs"] + else: " --parallel" # Phase 1 — frontend (nifler + `nim m`), run to a discovery fixpoint. var rounds = 0 diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 2b5ae6421e..a145458a90 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -245,6 +245,18 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode let canon = c.graph.canonTypes.getOrDefault(h) if canon != nil: op = getAttachedOp(c.graph, canon, kind) + if op == nil or op.ast.isGenericRoutine: + # IC: injectDestructorCalls is demand-driven and runs HERE (cg), not in the + # `lower` stage, so a structural, env-agnostic op the lower stage never had + # reason to serialize — most often a closure PROC type's `=destroy`/`=sink` + # (which act on the `(ClP_0, ClE_0)` tuple, NOT the concrete env) — must be + # lifted on demand, exactly as the lazy path's cg does. This is safe now: + # closure-env identity resolves via `attachedOps[itemId]`/env-erased typeKey, + # env objects load complete, and atomicRefOp's type-erased path covers any + # still-incomplete env (so the lift never walks a nil field). + excl t.flagsImpl, tfCheckedForDestructor + createTypeBoundOps(c.graph, nil, t, dest.info, c.idgen) + op = getAttachedOp(c.graph, t, kind) if op == nil: #echo dest.typ.id globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] & diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 9e00179fbb..3418bad729 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -176,7 +176,7 @@ proc closureParams(routine: PSym): PNode = result = routine.typ.n routine.ast[paramsPos] = result -proc addHiddenParam(routine: PSym, param: PSym) = +proc addHiddenParam*(routine: PSym, param: PSym) = assert param.kind == skParam var params = closureParams(routine) # -1 is correct here as param.position is 0 based but we have at position 0 diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 3684ec437a..0529812c3a 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -821,13 +821,15 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags # dynamic Acyclic refs need to use dyn decRef + let useStatic = isFinal(elemType) + let tmp = if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}: declareTempOf(c, body, x) else: x - if isFinal(elemType): + if useStatic: addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr)) var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType)) alignOf.typ = getSysType(c.g, c.info, tyInt) @@ -838,7 +840,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = var cond: PNode if isCyclic: - if isFinal(elemType): + if useStatic: let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) typInfo.typ = getSysType(c.g, c.info, tyPointer) cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo) @@ -873,7 +875,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: if isCyclic: - if isFinal(elemType): + if useStatic: let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) typInfo.typ = getSysType(c.g, c.info, tyPointer) body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y) diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 7c7756b0a9..d72b403a37 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -210,7 +210,62 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym = if matchesDerivedFieldId(n.sym.itemId, id): result = n.sym else: discard +proc lookupCapturedField(n: PNode, s: PSym): PSym = + ## Find an env field that `addField` would have produced for the captured + ## local `s`. Used as a fallback when the derived-itemId match fails because + ## `s` is a macro-generated gensym whose process-local id diverges from the + ## loaded env field's (see `addField`). `addField` always names a field + ## `s.name & $field.position`, so that pair uniquely identifies the field for a + ## local of this name without relying on the (unstable) item id. + result = nil + case n.kind + of nkRecList: + for i in 0..= 1 and n[0].kind == nkStrLit and - n[0].strVal in ["put", "inc", "add", "incl"]: - g.transitiveReplayActions.add n + # Record this transitively-loaded module so the sem driver applies its + # VM-level load effects (macro-cache replay + `{.compileTime.}` global init) + # exactly as for a direct import — see `pendingNifInit`. A throwaway module + # symbol (same shape as moduleFromNifFile's) gives the drain an idgen/info + # context; it is not registered, so a later direct import still loads fully. + if g.config.cmd == cmdM: + let m = PSym(kindImpl: skModule, itemId: itemId(int32(fileIdx), 0'i32), + name: getIdent(g.cache, splitFile(toFullPath(g.config, fileIdx)).name), + infoImpl: newLineInfo(fileIdx, 1, 1), positionImpl: int(fileIdx)) + setOwner(m, getPackage(g.config, g.cache, fileIdx)) + g.pendingNifInit.add (m, precomp.topLevel) + # Rebuild generic TYPE- and PROC-instance offers across the WHOLE closure, + # not just direct imports (`moduleFromNifFile`). An instance is frozen at + # the FIRST module to create it (in a scope where its body's symbols + # resolve unambiguously); a consumer many imports away must REUSE it rather + # than re-instantiate in its own scope, which may resolve a body symbol + # differently — a divergent `compiles()`-dependent array bound (SSZ + # `HashArray[8192, Gwei]`, type offer), or an ambiguous unqualified ident + # leaked from an unrelated import (`fromRaw` -> `SkRawPublicKeySize` from + # both `secp` and `secp256k1`, proc offer). Direct-only rebuild left the + # deep offer invisible when the clean instance lives a transitive hop away. + for off in precomp.typeOffers: + g.typeInstCache.mgetOrPut(off.generic.itemId, @[]).add off.inst + for off in precomp.genericOffers: + g.procInstCache.mgetOrPut(off.generic.itemId, @[]).add PInstantiation( + sym: off.inst, concreteTypes: off.concreteTypes, + genericParamsCount: off.genericParamsCount, compilesId: 0) for d in precomp.deps: stack.add d proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym = @@ -1053,6 +1083,14 @@ when not defined(nimKochBootstrap): sym: off.inst, concreteTypes: off.concreteTypes, genericParamsCount: off.genericParamsCount, compilesId: 0) + # Rebuild `typeInstCache` from this module's generic TYPE-instance OFFERS so a + # consumer's `searchInstTypes` reuses the baked instance (e.g. an SSZ + # `HashArray` whose array bound depends on import-scope-sensitive `compiles()`) + # rather than re-instantiating it with a divergent bound — see ast2nif's + # `(toffer …)`. Keyed by the generic body sym's itemId, as `searchInstTypes`. + for off in result.typeOffers: + g.typeInstCache.mgetOrPut(off.generic.itemId, @[]).add off.inst + # Mark module as cached g.cachedMods.incl fileIdx.int g.hookClosure.incl fileIdx.int @@ -1084,6 +1122,10 @@ when not defined(nimKochBootstrap): # walks the closure in nifbackend.loadModuleDependencies.) if g.config.cmd == cmdM: loadTransitiveHooks(g, result.deps) + # Record the directly-loaded module for the same VM-level load effects as its + # transitive deps (`pendingNifInit`). AFTER loadTransitiveHooks so the drain + # applies deps before the dependent (macro-cache order). + g.pendingNifInit.add (m, result.topLevel) proc isModuleFile(g: ModuleGraph; fileIdx: FileIndex): bool = let i = fileIdx.int32 diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index 4f7791fe15..9e90aa05e8 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -26,6 +26,7 @@ import ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif, typekeys, cnif from cgmeth import generateIfMethodDispatchers +from transf import transformBody import ic / replayer proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex; @@ -139,12 +140,57 @@ proc signatureHasMetaType(t: PType; depth: int = 0): bool = # as meta and drop it from the owned-routine seeding -> undefined symbols # at link (its only definer never emits it). return false - if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyStatic, tyGenericParam, + if t.kind == tyStatic: + # A RESOLVED static value (the `256` in `MDigest[256]`, the `N` in + # `HashList[T, N]`, …) is carried as a `tyStatic` node inside the otherwise + # fully-concrete `tyGenericInst`, but it is NOT meta: the routine is a normal + # runtime routine the owner must emit. Only an UNRESOLVED `static T` parameter + # (no bound value, `t.n == nil`) is meta. Without this, every routine whose + # signature touches a `static`-parameterized generic instance (the bulk of + # the SSZ/`MDigest` API) is dropped from the owned-routine seeding and ends up + # an undefined reference at link (mirrors the tyGenericBody case above). + return t.n == nil + if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyGenericParam, tyAnything, tyFromExpr, tyError}: return true for k in t.kids: if signatureHasMetaType(k, depth + 1): return true +proc ownsRuntimeRoutine(s: PSym; modPos: int): bool = + ## A concrete, non-generic, runtime routine with a real body, OWNED by the + ## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a + ## routine called only from other modules is still emitted by somebody) and + ## the `lower` stage's owned-routine enumeration, so both stages see exactly + ## the same set. The exclusions: + ## - nested/closure procs (owner is a proc, not a module): emitted via their + ## enclosing routine's lambda-lifting, never standalone; + ## - generic instances (`sfFromGeneric`): emitted by demand, deduped by merge; + ## - `importc`/`compileTime`/`error`/forward sentinels and meta signatures: + ## not real codegen targets. + ## - method DISPATCHERS (`sfDispatcher`): their bodies are (re)synthesized into + ## the main TU by `emitMethodDispatchers`/`generateIfMethodDispatchers`, never + ## per module. A dispatcher is a `copySym` clone of the method that shares the + ## method's body sub-tree (incl. its closure iterator); transforming it here + ## would lambda-lift that SHARED iterator a SECOND time under a different owner + ## identity, baking a conflicting `up` field → "up references do not agree" + ## (the divergence is impossible in non-IC, where the dispatcher body is empty + ## at lift time). So a dispatcher is never an owned runtime routine. + ## A `{.closure.}` iterator IS a standalone runtime routine (unlike an inline + ## iterator, which is expanded at each call site) and must be emitted by its + ## owner — else a cross-module `for` over it links to nothing. + s.itemId.module == modPos and + (s.kind in {skProc, skFunc, skConverter, skMethod} or + (s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and + s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and + s.magic == mNone and + sfFromGeneric notin s.flags and + sfDispatcher notin s.flags and + {sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and + s.typ != nil and not signatureHasMetaType(s.typ) and + s.ast != nil and s.ast.safeLen > bodyPos and + s.ast[genericParamsPos].kind == nkEmpty and + s.ast[bodyPos].kind != nkEmpty + proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) = ## Generate C code for a single module. let moduleId = precomp.module.position @@ -170,34 +216,7 @@ proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) = if g.config.cmd == cmdNifC and g.config.icBackendStage == "cg": let modPos = precomp.module.position for s in moduleSymbolStubs(ast.program, FileIndex modPos): - if s.itemId.module == modPos and - s.kind in {skProc, skFunc, skConverter, skMethod} and - # Only MODULE-level routines: a nested/closure proc (its owner is a - # proc) captures its enclosing scope and cannot be emitted standalone — - # the captured params have no loc → `expr: param not init`. Nested procs - # are emitted via their enclosing routine's lambda-lifting, so seeding - # the enclosing (module-level) routine already covers them. - s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and - s.magic == mNone and - # Skip generic instances: they have no single owning-module top-level - # and are emitted by demand (emit-everywhere, deduped by the merge - # stage). An instance has an empty `genericParamsPos` just like a plain - # concrete proc, so only `sfFromGeneric` tells them apart; seeding one - # would force standalone codegen of an instance body whose `when T is X` - # branches were never folded for this path → `genMagicExpr: mIs`. - sfFromGeneric notin s.flags and - # Every other routine the module owns must be emitted here, exported or - # not: a non-exported helper is still reached from another module when a - # `template`/inline routine expands at a call site there (e.g. msgs' - # `internalErrorImpl` behind the `internalError` template), and that - # caller now only prototypes it. `{.error.}`/`compileTime` sentinels and - # bodyless forward decls are not real codegen targets. - {sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and - s.typ != nil and not signatureHasMetaType(s.typ) and - s.ast != nil and s.ast.safeLen > bodyPos and - s.ast[genericParamsPos].kind == nkEmpty and - s.ast[bodyPos].kind != nkEmpty: - # a concrete, non-generic, runtime routine with a real body, owned here + if ownsRuntimeRoutine(s, modPos): requestProcDef(bmod, s) proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex): @@ -220,6 +239,16 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex): g.config.m.systemFileIdx = systemFileIdx var precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface}) g.systemModule = precompSys.module + if precompSys.module != nil: + # The precompiled-load path does not restore `sfSystemModule` (mirror of the + # `sfMainModule` re-add above). `registerReusedModuleToMain` keys on it to put + # the system module's init right after its datInit AND to emit + # `initStackBottomWith` into `mainDatInit` — so that the main thread's stack + # bottom is set before any module's init runs. Without the flag the system + # init is mis-routed into the regular `otherModsInit` bucket and + # `initStackBottomWith` is never registered, so a GC cycle during a module's + # init (under refc) scans the stack with a nil bottom and crashes. + incl precompSys.module.flagsImpl, sfSystemModule var nifFiles: seq[string] = @[toNifFilename(g.config, systemFileIdx)] var modules = loadModuleDependencies(g, mainFileIdx, nifFiles, depFlags = {}) # loadModuleDependencies traverses the project's import closure and stops at @@ -323,6 +352,170 @@ proc findTargetModule(g: ModuleGraph; modules: seq[PrecompiledModule]; cachedModuleSuffix(g.config, FileIndex precompSys.module.position) == suffix: return precompSys +proc setNestedClosureBodies(g: ModuleGraph; idgen: IdGenerator; n: PNode; + owner: PSym; seen: var IntSet) = + ## A closure routine nested in `owner` (the `:anonymous` proc lambda-lifting + ## minted, plus any deeper nesting) gets its captured-var→env rewrite produced + ## as part of the OWNER's `transformBody`. The nested proc is a module-indexed + ## sym whose `.s.nif` sdef carries its PRE-lift body, so without help the whole + ## module re-serializer would write that pre-lift body and cg would lose the + ## capture mapping (it accesses `x` directly instead of `ClE_0->x0`). Walk the + ## owner's transformed body and cache each nested closure's transformed body on + ## its sym so `writeSymDef` serializes the lifted body into the routine's + ## 2-way-body slot. + if n == nil: return + if n.kind == nkSym: + let s = n.sym + if s != nil and s.kind in routineKinds and s != owner and + not seen.containsOrIncl(s.id): + if s.ast != nil and getBody(g, s).kind != nkEmpty and + s.typ != nil and s.typ.callConv == ccClosure: + if s.transformedBody == nil: + s.transformedBody = transformBody(g, idgen, s, {}) + setNestedClosureBodies(g, idgen, s.transformedBody, s, seen) + else: + for i in 0 ..< n.safeLen: + setNestedClosureBodies(g, idgen, n[i], owner, seen) + +proc reownFromTwin(n: PNode; twin, s: PSym) = + ## Re-own to `s` every entity the frontend attributed to `s`'s forward-decl + ## `twin` (found via the result's owner). lambda-lifting compares owners by + ## reference, so a twin-owned `result` is rejected as `illegalCapture` + ## ("'result' ... cannot be captured") and, once that is fixed, twin-owned + ## locals go missing from `s`'s env ("environment misses: ..."). Both are + ## pervasive on chronos `{.async.}` methods. Re-owning to `s` matches the + ## single-sym non-IC case. `twin` is ONE specific sym, so only THIS routine's + ## result-twin-owned entities match — re-owning entities of OTHER same-name + ## twins proved too blunt (it disrupts env construction and reintroduces the + ## very capture errors it should fix). `n.sym != s` guards self-ownership. + if n == nil: return + if n.kind == nkSym and n.sym != nil and n.sym != s and n.sym.owner == twin: + setOwner(n.sym, s) + for i in 0 ..< n.safeLen: + reownFromTwin(n[i], twin, s) + +proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) = + ## Per-module backend lowering (`--icBackendStage:lower --icBackendModule:`): + ## enumerate the routines this module OWNS and write them to `.t.nif`. + ## Eventually this transforms each owned routine once, in the owner's id space, + ## so `cg` reads the result instead of re-deriving it (re-derivation per + ## parallel `cg` process is the root of the closure-`:env` identity drift). + ## Runs per module in parallel on the shallow backend dep-graph — NOT folded + ## into the dense, mostly-serial sem stage. + ## + ## gate `newSymNode`'s lazy-type marking to the backend (see astdef) — the + ## transform builds sym nodes off not-yet-typed stubs, exactly as the `cg` + ## stage does. + nifcBackendActive = true + let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx) + let targetIsMain = g.config.icBackendModule.len == 0 or + g.config.icBackendModule == mainSuffix + var modules: seq[PrecompiledModule] + var precompSys: PrecompiledModule + var target: PrecompiledModule + if targetIsMain: + var nifFiles: seq[string] + (modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx) + if modules.len == 0: + rawMessage(g.config, errGenerated, + "Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx)) + return + target = findTargetModule(g, modules, precompSys, g.config.icBackendModule) + else: + (modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule) + if target.module == nil: + rawMessage(g.config, errGenerated, + "per-module lowering: module not found for suffix: " & g.config.icBackendModule) + return + let modPos = target.module.position + let tb = BModuleList(g.backend).mods[modPos] + if tb == nil: + rawMessage(g.config, errGenerated, + "per-module lowering: no backend module for suffix: " & g.config.icBackendModule) + return + # Transform every owned routine ONCE in this single process's id space and + # re-serialize the ENTIRE module as a proper indexed NIF (`writeLoweredModule`) + # with the transformed bodies baked into the routine `(sd)` entries. `cg` loads + # it through the normal module loader, so nested procs (incl. async state + # machines) arrive as real defs with their lifted bodies — no re-derivation. + # This single-writer-per-owner is what keeps closure-`:env` identity stable + # across the parallel `cg` processes (re-derivation per process was the root of + # the `:env` identity drift). `transformBody` with flags {} mirrors the cg call + # (cgen.nim); `injectDestructorCalls` is NOT run here — it stays in `cg` on the + # loaded body. + # + # `transformBody`/lambda-lifting LIFTS the closure env's type-bound ops + # (`=destroy` etc.) into `g.opsLog`; snapshot its length so we serialize exactly + # the ops THIS stage created (not those loaded from `.s.nif`). + let opsLogStart = g.opsLog.len + for s in moduleSymbolStubs(ast.program, FileIndex modPos): + if ownsRuntimeRoutine(s, modPos): + # `.s.nif` wins: a routine already transformed during sem (CT eval / macro / + # VM transform) carries its lowered body in the `.s.nif` slot — don't + # re-transform it here. + if s.transformedBody != nil: continue + # A routine serialized as a forward-decl + impl pair (writeSymDef's + # "separate forward declaration and implementation") loads as TWO syms; the + # impl `s` we transform here can carry body entities (`result`, locals, + # nested routines) owned by its fwd-decl TWIN, not by `s`. lambda-lifting + # compares owners by reference → `illegalCapture` rejects a twin-owned + # `result` and the lifting pass can't find twin-owned locals in `s`'s env. + # Pervasive on chronos `{.async.}` methods. Re-own them to `s`, matching the + # single-sym non-IC case. Backend-only, so frontend effect/exception + # inference is untouched. + if s.ast != nil and s.ast.len > resultPos and + s.ast[resultPos].kind == nkSym and s.ast[resultPos].sym.owner != s: + reownFromTwin(s.ast, s.ast[resultPos].sym.owner, s) + # Retain the transformed body on the sym so `writeSymDef` serializes it in + # the routine's `(sd)` 2-way-body slot. + s.transformedBody = transformBody(g, tb.idgen, s, {}) + # Cache the lifted body on nested ccClosure routines too, so a module-indexed + # nested closure serializes its lifted (capture-rewritten) body. + var seenNested = initIntSet() + setNestedClosureBodies(g, tb.idgen, s.transformedBody, s, seenNested) + # Collect the hooks this stage lifted, and transform each hook ROUTINE's body + # too (it is itself lowered into NIFC). The hooks' `(sd)` + transformed body go + # into the `.t.nif`; `cg` re-attaches them so `injectDestructorCalls` resolves + # the loaded env's `=destroy`. Iterate to a fixpoint: a hook body can lift + # further hooks (a field's `=destroy`). + var hooks: seq[LogEntry] = @[] + var i = opsLogStart + while i < g.opsLog.len: + let e = g.opsLog[i] + if e.kind == HookEntry and e.sym != nil and e.sym.kind in routineKinds and + e.sym.transformedBody == nil: + hooks.add e + # Transform the hook routine's body and cache it on the sym so `writeSymDef` + # serializes it in the hook's `(sd)` transformed-body slot (`transformBody + # {}` returns the body but does not cache it). + e.sym.transformedBody = transformBody(g, tb.idgen, e.sym, {}) + inc i + # Re-serialize the whole module to its suffix-based `.t.nif` (the path + # `toNifFilename` resolves for the cg/emit stages). `writeLoweredModule` seals + # routines itself. + let suffix = cachedModuleSuffix(g.config, FileIndex modPos) + let wholeArtifact = toGeneratedFile(g.config, AbsoluteFile(suffix), ".t.nif").string + writeLoweredModule(ast.program, g.config, target, hooks, wholeArtifact) + if isDefined(g.config, "icDceCheck"): + stderr.writeLine "[icLower] " & extractFilename(wholeArtifact) & " " & + $hooks.len & " hooks" + +proc visitDep(suffix: string; + suffixToMod: Table[string, PrecompiledModule]; + visited: var HashSet[string]; bl: BModuleList; + ordered: var seq[BModule]) = + ## Post-order DFS over a module's import closure used to reconstruct the + ## dependency (init) order: a dependency's init must be registered before its + ## importer's. Appends each reachable non-main module's `BModule` to `ordered`. + if visited.containsOrIncl(suffix): return + let pm = suffixToMod.getOrDefault(suffix) + if pm.module == nil: return + for dep in pm.deps: # dependencies first (post-order) + visitDep(dep.string, suffixToMod, visited, bl, ordered) + if sfMainModule notin pm.module.flags: + let bm = bl.mods[pm.module.position] + if bm != nil: ordered.add bm + proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = ## Per-module backend codegen (`--icBackendStage:cg --icBackendModule:`): ## generate C for the single module named by `icBackendModule` and write only @@ -336,6 +529,8 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = ## module still loads everything (`loadBackendModules`) because NimMain's init ## list and the method dispatchers are whole-program; its `cg` runs essentially ## alone (every other `.c.nif` precedes it), so it does not contend for memory. + # gate `newSymNode`'s lazy-type marking to this stage only (see astdef) + nifcBackendActive = true let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx) let targetIsMain = g.config.icBackendModule.len == 0 or g.config.icBackendModule == mainSuffix @@ -363,6 +558,10 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = "per-module codegen: module not found for suffix: " & g.config.icBackendModule) return + # The `lower` stage already wrote each module's transformed bodies + lifted + # hooks into its `.t.nif`, which the loaders above read directly (toNifFilename + # resolves the `.t.nif`); transformed bodies arrive via loadSymFromCursor and + # lifted hooks via moduleFromNifFile's registerLoadedHooks. Nothing to apply. generateCodeForModule(g, target) let bl = BModuleList(g.backend) # The main module also owns the whole-program method dispatchers + NimMain. @@ -373,10 +572,58 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = # `cg` processes, so the calls are registered here from each `.c.nif` meta # head — which is why the main module's `cg` runs last, after every other # `.c.nif` exists. Modules without init code (no `.c.nif`) register nothing. + # + # The registration order IS the runtime init order, and it must be the + # DEPENDENCY (post-order) order: an imported module's init has to run before + # its importer's. The whole-program backend gets this for free — it iterates + # `modulesClosed`, built in module-FINISH order (a post-order DFS over + # imports). Iterating `bl.mods` by position is WRONG: an importer gets a + # LOWER position than the modules it imports (its file is registered before + # its `import` statements are processed), so position order runs importers + # before their dependencies. That left chronicles' `topics_registry` — whose + # init sets `mainThreadId` — running AFTER a module that calls `registerTopic` + # from its own init, tripping the `getThreadId() == mainThreadId` assert at + # startup. So reconstruct the post-order DFS over the import closure here. + # + # NOTE: this is deliberately a SEPARATE traversal rather than reusing the + # module LOAD order — the per-module backend's C emit is sensitive to load + # order (it determines the main TU's header composition), so the loader must + # keep its existing order and the init order is derived independently here. + var suffixToMod = initTable[string, PrecompiledModule]() + for pm in modules: + if pm.module != nil: + suffixToMod[cachedModuleSuffix(g.config, FileIndex pm.module.position)] = pm + if precompSys.module != nil: + suffixToMod[cachedModuleSuffix(g.config, FileIndex precompSys.module.position)] = precompSys + var visited = initHashSet[string]() + var ordered: seq[BModule] = @[] + # System (and its include/import closure) must initialize FIRST: its init + # runs `initGC()` (top-level code in `threadimpl`, included into system), + # and every other module's init may allocate — an allocation before the GC + # heap is set up triggers a collection over an uninitialized region and + # crashes (e.g. nim-metrics' `newRegistry` in its init). System is the + # IMPLICIT universal import and appears in no module's explicit `deps`, so a + # DFS rooted at main never reaches it; seed the traversal from system first. + if precompSys.module != nil: + visitDep(cachedModuleSuffix(g.config, FileIndex precompSys.module.position), + suffixToMod, visited, bl, ordered) + # Then order the whole import closure rooted at the main module; main itself + # is excluded above (its init body becomes NimMain). + for pm in modules: + if pm.module != nil and sfMainModule in pm.module.flags: + visitDep(cachedModuleSuffix(g.config, FileIndex pm.module.position), + suffixToMod, visited, bl, ordered) + # Defensive: any loaded module not reachable from main's import closure + # (demand-loaded system internals) keeps its init registered, appended last + # — nothing imports it, so its relative order does not matter. for m in bl.mods: if m != nil and sfMainModule notin m.module.flags: - let heads = readCnifHeads(getCFile(m).string & ".nif") - registerReusedModuleToMain(bl, m, heads.initRequired, heads.datInitRequired) + let suffix = cachedModuleSuffix(g.config, FileIndex m.module.position) + if not visited.containsOrIncl(suffix): + ordered.add m + for m in ordered: + let heads = readCnifHeads(getCFile(m).string & ".nif") + registerReusedModuleToMain(bl, m, heads.initRequired, heads.datInitRequired) let tb = bl.mods[target.module.position] if tb != nil: finishModule(g, tb) @@ -457,7 +704,17 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) = let artifact = cfile & ".nif" var dropped = 0 let code = renderCFromArtifact(artifact, decision, extractFilename(artifact), dropped) - writeFile(cfile, code) + # Write the `.c` content-stably. `merge` re-runs on any edit and bumps the + # decision file's mtime, so nifmake re-fires every `emit` (the filter is cheap); + # but the FILTERED output is usually byte-identical for modules unaffected by + # the edit. Rewriting it unconditionally would bump every `.c`'s mtime and make + # `callCCompiler` recompile every `.o`. Writing only on a real change preserves + # the mtime, so the C compiler recompiles exactly the modules whose `.c` changed + # — the same DCE model as Nimony's. Safe here (unlike a content-stable merge + # decision): a `.c` is a per-module LEAF consumed only by the C compiler's own + # up-to-date check, not a shared prerequisite in nifmake's mtime ordering. + if not fileExists(cfile) or readFile(cfile) != code: + writeFile(cfile, code) if isDefined(g.config, "icDceCheck"): stderr.writeLine "[icEmit] " & extractFilename(cfile) & " dropped " & $dropped & " bodies (" & $code.len & " bytes)" @@ -483,6 +740,7 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) = if precompSys.module != nil: replayBackendActions(g, precompSys.module, precompSys.topLevel) let bl = BModuleList(g.backend) + var addedCFiles = initHashSet[string]() for m in bl.mods: if m != nil: let cfile = getCFile(m) @@ -490,17 +748,53 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) = # (extra members of system's closure that no build rule targets) had their # code emit-everywhere'd into the targets, so they have no file to compile. if not fileExists(cfile.string): continue + addedCFiles.incl extractFilename(cfile.string) var cf = Cfile(nimname: m.module.name.s, cname: cfile, obj: completeCfilePath(g.config, toObjFile(g.config, cfile)), flags: {}) - addFileToCompile(g.config, cf) + # `addExternalFileToCompile` (not `addFileToCompile`) gates each `.c` on its + # SHA1 footprint: an unchanged `.c` keeps its `.o` and is flagged Cached, so + # `callCCompiler` skips its compile but still links the existing object. This + # is what makes a localized edit recompile only the handful of `.c`s the + # `emit` stage actually rewrote, instead of every object every time — the + # final piece of per-module backend incrementality after the merge barrier. + addExternalFileToCompile(g.config, cf) + # deps.nim's static scanner can keep a CONDITIONALLY-imported module as a build + # node (e.g. `net`'s `when defineSsl: import openssl`, or a `when defined(os)` + # import) that the NIF-`deps` walk above never reaches because the condition is + # off. Such a node still emitted a `.c`, and it can OWN a live generic instance + # that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused by + # `strutils.escape`) — so its body must be at link or that reference is + # undefined. Link every emitted `.c` the merge decision says OWNS a LIVE symbol; + # a node that owns nothing live (a Windows-only winsock node on Linux) is + # correctly skipped. + block: + let nimcache = getNimcacheDir(g.config).string + let decision = readMergeDecision(nimcache / MergeDecisionFile) + if not decision.broken: + var liveOwners = initHashSet[string]() + for cname, owner in decision.owners: + if owner.endsWith(".c.nif") and cname in decision.live: + liveOwners.incl owner + for owner in liveOwners: + let cbase = owner[0 ..< owner.len - ".nif".len] # "@m….nim.c.nif" -> ".c" + if addedCFiles.containsOrIncl(cbase): continue + let cfile = AbsoluteFile(nimcache / cbase) + if not fileExists(cfile.string): continue + var cf = Cfile(nimname: cbase, cname: cfile, + obj: completeCfilePath(g.config, toObjFile(g.config, cfile)), + flags: {}) + addExternalFileToCompile(g.config, cf) if g.config.cmd != cmdTcc: extccomp.callCCompiler(g.config) proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = ## Main entry point for NIF-based C code generation. ## Traverses the module dependency graph and generates C code. - if g.config.icBackendStage == "cg": + if g.config.icBackendStage == "lower": + generateLowerStage(g, mainFileIdx) + return + elif g.config.icBackendStage == "cg": generateCgStage(g, mainFileIdx) return elif g.config.icBackendStage == "merge": @@ -514,4 +808,4 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = return else: rawMessage(g.config, errGenerated, - "the per-module NIF backend requires --icBackendStage:cg|merge|emit|link") + "the per-module NIF backend requires --icBackendStage:lower|cg|merge|emit|link") diff --git a/compiler/options.nim b/compiler/options.nim index 73fc0a7596..42c2aee49f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -29,7 +29,7 @@ const nimEnableCovariance* = defined(nimEnableCovariance) - icFormatVersion* = "6" + icFormatVersion* = "21" ## Version of the IC cache format (the sem-NIF module layout written by ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever ## that layout changes: `commandIc` wipes a nimcache whose `ic.version` diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index a71ecddac9..d9770292cc 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -298,6 +298,16 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator if not hasNil: genericOffers.add (inst.sym.instantiatedFrom, inst.sym, inst.concreteTypes, inst.genericParamsCount) + # Generic TYPE-instance OFFERS: every `tyGenericInst` THIS module created, + # so a consumer reuses its baked structure (array bounds etc.) rather than + # re-instantiating with a scope-divergent bound. See ast2nif.writeNifModule. + var typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[] + for genItemId, instList in graph.typeInstCache: + for inst in instList: + if inst != nil and inst.uniqueId.module == module.position and + inst.kidsLen > 0 and inst[0] != nil and + inst[0].kind == tyGenericBody and inst[0].sym != nil: + typeOffers.add (inst[0].sym, inst) # The module's REAL resolved direct imports (incl. macro/template-generated # ones with no surviving syntactic node). Passed to writeNifModule so the # NIF `deps` section is complete (the backend closure walk needs it), and @@ -305,7 +315,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator let resolvedImportDeps = graph.importDeps.getOrDefault(module.position.FileIndex, @[]) writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, replayActions, implDeps, reexportedModuleSyms(graph, module), - genericOffers, resolvedImportDeps) + genericOffers, typeOffers, resolvedImportDeps) # The module's REAL direct imports (incl. macro-generated) for `nim ic`'s # graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep. var semDepPaths: seq[string] = @[] @@ -351,6 +361,31 @@ proc initLoadedCompileTimeGlobals(graph: ModuleGraph; module: PSym; topLevel: PN sect.add s.ast setupCompileTimeVar(module, idgen, graph, sect) +proc finalizeLoadedModules(graph: ModuleGraph) = + ## Apply the VM-level load effects of every module just loaded from a NIF — + ## direct import OR dep-of-a-dep, both collected in `graph.pendingNifInit` by the + ## loader (modulegraphs.moduleFromNifFile / loadTransitiveHooks). This is the ONE + ## place that knows what loading a module does to global VM state, so a + ## transitively-reached module (which never passes through this proc's caller) + ## gets identical treatment. Modules are in dependency order (deps before + ## dependents), which is the correct macro-cache replay order. + ## 1. macro-cache replay: std/macrocache put/inc/add/incl recorded in the + ## module's top level (pragma replay actions are a backend concern, skipped). + ## 2. eager `{.compileTime.}` global init (see initLoadedCompileTimeGlobals). + ## To add a new per-load effect, extend this proc — do not add a parallel buffer. + if graph.pendingNifInit.len == 0: return + for (m, topLevel) in graph.pendingNifInit: + if topLevel == nil: continue + var replayList = newNodeI(nkStmtList, m.info) + for n in topLevel: + if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and + n[0].strVal in ["put", "inc", "add", "incl"]: + replayList.add n + if replayList.len > 0: + replayStateChanges(m, graph, replayList) + initLoadedCompileTimeGlobals(graph, m, topLevel) + graph.pendingNifInit.setLen 0 + proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags; fromModule: PSym = nil): PSym = var flags = flags if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule @@ -420,33 +455,12 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF if sfSystemModule in flags: graph.systemModule = result partialInitModule(result, graph, fileIdx, AbsoluteFile(toFullPath(graph.config, fileIdx))) - # Replay the module's recorded state changes: macro-cache operations - # (std/macrocache puts/incs/adds/incls) plus a few pragmas. The loader - # parsed them into `precomp.topLevel` (mixed with other top-level nodes), - # so filter to the replay actions. A loaded module's `ast` is never - # rebuilt, so this used to be skipped (`result.ast == nil`) and a - # NIF-loaded module's macro cache was lost — e.g. nim-serialization's - # flavor registration became invisible to dependents (`DefaultFlavor: - # automatic serialization is not enabled`). - var replayList = newNodeI(nkStmtList, result.info) - for n in precomp.topLevel: - # Only macro-cache ops (put/inc/add/incl). The pragma replay actions - # (compile/link/passc/hint/...) are a backend/link concern handled by - # the nifc closure, and re-emitting a loaded module's hints/warnings on - # every import would be wrong — so they are deliberately skipped here. - if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and - n[0].strVal in ["put", "inc", "add", "incl"]: - replayList.add n - # Plus the macro-cache actions of the module's transitive import closure - # (collected by the moduleFromNifFile call above via loadTransitiveHooks), - # so a flavor/type registered in an indirectly-imported module is visible. - for n in graph.transitiveReplayActions: replayList.add n - graph.transitiveReplayActions.setLen 0 - if replayList.len > 0: - replayStateChanges(result, graph, replayList) - # Fill the VM slots of the module's `{.compileTime.}` globals now (sem - # would have, but a NIF-loaded module is never semchecked). - initLoadedCompileTimeGlobals(graph, result, precomp.topLevel) + # Apply the VM-level load effects of this module AND every dep it pulled in + # (moduleFromNifFile recorded them all in graph.pendingNifInit): macro-cache + # replay (else a NIF-loaded module's macro cache is lost — e.g. + # nim-serialization flavor registration) and eager `{.compileTime.}` global + # init. Uniform for direct and transitive deps — see finalizeLoadedModules. + finalizeLoadedModules(graph) return result # Return early, don't process from source let path = toFullPath(graph.config, fileIdx) let filename = AbsoluteFile path @@ -551,6 +565,11 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx localError(graph.config, unknownLineInfo, "nim m requires precompiled NIF for system module (expected: " & nifPath & ")") return + # Apply system's (and its deps') load effects now: the main module is + # compiled from source and never re-enters the moduleFromNifFile drain for + # system, so without this its macro-cache / CT globals would wait until the + # first NIF import is processed. See finalizeLoadedModules. + finalizeLoadedModules(graph) discard graph.compilePipelineModule(projectFile, {sfMainModule}) else: graph.compilePipelineSystemModule() diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 0d32930472..8c740f54bf 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1117,6 +1117,45 @@ proc trackCall(tracked: PEffects; n: PNode) = #if canRaise(a): # echo "this can raise ", tracked.config $ n.info let op = a.typ + # A routine whose body reaches a compile-time-only magic (`macros.error`, + # `slurp`, `gorge`, `getAst`, …) can never be code-generated — the C/JS + # backends reject those magics (ccgexprs `errXMustBeCompileTime`). Such a + # routine is compile-time-only by construction; mark it `sfCompileTime` so it + # is treated uniformly as such. Non-IC pruned it by demand-driven codegen, but + # the per-module IC backend emits every owned routine (no DCE) and would + # otherwise feed the magic to codegen. Mirrors the `tfTriggersCompileTime -> + # sfCompileTime` path in `semProcAux`. + if a.kind == nkSym and a.sym.magic in {mNLen..mNError, mSlurp..mQuoteAst} and + tracked.owner != nil and tracked.owner.kind in routineKinds and + tracked.config.cmd != cmdNimscript: + # ...but NOT under `nim e`: nimscript has no codegen backend to protect, and + # marking a routine `sfCompileTime` makes `semExpr` eagerly fold calls to it + # at sem time (emConst), where module-level globals it reads have no VM slot + # yet — distros' `detectOsWithAllCmd` reaches `gorge` and reads the plain + # global `unameRes` → "cannot evaluate at compile time: unameRes". In the + # normal nimscript run (emRepl) the module's var section runs first and the + # slot exists, so the marking is both unnecessary and harmful here. + # + # ...and NOT if the routine is — or is nested inside — a macro/template: + # those are VM-only (never code-generated), so the per-module IC backend has + # nothing to protect there, while `sfCompileTime` on a macro-internal nested + # closure breaks its captured-variable access in the VM ("cannot evaluate at + # compile time: n" — `tests/macros/tmacros1`'s `innerProc` reading the + # macro-local `n`). Walk the owner chain and bail on the first + # skMacro/skTemplate. NB mark `tracked.owner` (the routine that directly + # reaches the magic), NOT its outermost enclosing: a runtime proc may legally + # nest a compile-time helper — `tests/generics/tunique_type`'s `[]` proc + # contains a nested `buildResult` macro — and marking the proc would wrongly + # make IT compile-time ("request to generate code for .compileTime proc: []"). + var encl = tracked.owner + var insideMeta = false + while encl != nil and encl.kind != skModule: + if encl.kind in {skMacro, skTemplate}: + insideMeta = true + break + encl = encl.skipGenericOwner + if not insideMeta: + incl(tracked.owner, sfCompileTime) if n.typ != nil: if tracked.owner.kind != skMacro and n.typ.skipTypes(abstractVar).kind != tyOpenArray: createTypeBoundOps(tracked, n.typ, n.info) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 04bc2e3ee5..ab3a30a29e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2621,6 +2621,17 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, addParams(c, proto.typ.n, proto.kind) proto.info = s.info # more accurate line information proto.options = s.options + # `s` (the impl symbol) is discarded in favour of `proto`. It still carries + # `s.ast == n` (set above) and stays reachable as the owner of body-local + # symbols, so under IC it would be serialized as a SECOND, body-bearing + # `proc` entry — a phantom duplicate of `proto`. The per-module backend then + # codegens that phantom, whose `result` is owned by `proto` (addResult below + # re-parents it), not by the phantom: lambdalifting's capture check + # (`result.skipGenericOwner != owner`) then wrongly classifies `result` as a + # captured outer variable → "'result' … cannot be captured". Drop the + # discarded impl's body so it can never be emitted as a routine (same leak + # class the `miscPos` adoption below guards against for generic params). + let discardedImpl = s s = proto n[genericParamsPos] = proto.ast[genericParamsPos] n[paramsPos] = proto.ast[paramsPos] @@ -2638,6 +2649,19 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if importantComments(c.config) and proto.ast.comment.len > 0: n.comment = proto.ast.comment proto.ast = n # needed for code generation + if discardedImpl != proto: + discardedImpl.ast = nil + # The impl symbol is discarded in favour of `proto`, but it stays `Complete` + # in this module, so `ast2nif.shouldWriteSymDef` still serializes it. With + # `sfExported` it would be written importable (`x` marker) and an importer + # would load BOTH it and `proto` into the overload set: "ambiguous call; + # both foo and foo" (identical signatures). Normally a discarded impl is a + # gensym/transient that isn't reached this way, but a `{.async: (raises).}` + # forward-decl + impl reconciles HERE with both syms exported. Strip the + # export so the design's "forward declarations are never importable" holds — + # the def still serializes (other refs may resolve to it) but is invisible + # to importer overload resolution; `proto` carries the export. + excl(discardedImpl, sfExported) popOwner(c) pushOwner(c, s) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 49b4f781d6..9a230faf83 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -480,7 +480,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = else: header = instCopyType(cl, t) - result = newType(tyGenericInst, cl.c.idgen, t.genericHead.owner, son = header.genericHead) + # The instantiating module owns the instance (and announces it as an offer): + # the generic body's module (`t.genericHead.owner`) has no business owning a + # type that references instantiation-site types — that is the IC parent->child + # heap leak the write-barrier surfaces. + result = newType(tyGenericInst, cl.c.idgen, cl.c.module, son = header.genericHead) result.flags = header.flags # be careful not to propagate unnecessary flags here (don't use rawAddSon) # ugh need another pass for deeply recursive generic types (e.g. PActor) @@ -834,15 +838,21 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType, isInstValue = false): # trough replaceObjBranches in order to resolve any pending nkRecWhen nodes result = t - # Slow path, we have some work to do - if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and t.elementType.n != nil: + # Slow path, we have some work to do. CRUCIAL: only ever mutate a type that + # is LOCAL to the module we are instantiating in (`uniqueId.module == + # idgen.module`). A type loaded from another module's NIF (foreign) already + # had its object branches resolved when it was originally compiled; mutating + # it in place here is an old→new heap write that re-homes the loaded type to + # the instantiation site (its sym then looks owned by the consumer module and + # loses its `info`, colliding C type names — the libp2p `Message` bug). The + # prior `state != Sealed` guard was insufficient: a freshly-LOADED type is + # `Complete`, not `Sealed` (`Sealed` only means "already re-written to a NIF"). + if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and + t.elementType.n != nil and t.elementType.uniqueId.module == cl.c.idgen.module.int: discard replaceObjBranches(cl, t.elementType.n) - elif result.n != nil and t.kind == tyObject and result.state != Sealed: - # A type loaded from the IC cache already had its object branches - # resolved when it was originally compiled, and must not be mutated in - # place (nor copied, which would break object-inheritance identity), so - # only non-Sealed types are processed here. + elif result.n != nil and t.kind == tyObject and result.state != Sealed and + result.uniqueId.module == cl.c.idgen.module.int: # Invalidate the type size as we may alter its structure result.size = -1 result.n = replaceObjBranches(cl, result.n) diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 7705ff62d9..08c83ad29e 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -10,6 +10,7 @@ ## Computes hash values for routine (proc, method etc) signatures. import ast, ropes, modulegraphs, options, msgs, pathutils +from lineinfos import FileIndex from std/hashes import Hash import std/tables import types @@ -74,7 +75,19 @@ proc hashTypeSym(c: var MD5Context, s: PSym; conf: ConfigRef) = c &= ":anon" else: var it = s - c &= customPath(conf.toFullPath(s.info)) + # The source file path disambiguates same-named object types from different + # modules whose owner-chain names also coincide (e.g. libp2p kademlia/protobuf + # `Message` vs rendezvous/protobuf `Message`, both modules named `protobuf`). + # A type sym that reaches the backend as a `Complete` stub never individually + # loaded carries `unknownLineInfo` (fileIndex -1), which `toFullPath` collapses + # to the `???` placeholder — so the two would hash to ONE mangled C name and the + # wrong struct gets emitted. Fall back to the sym's HOME module file (its + # per-module NIF-suffix path, stable+unique) for the path. Only fires on a -1 + # fileIndex; non-IC type syms always have a real `info`, so the fast path is + # taken and the hash is unchanged (koch boot byte-equal). + let infoFi = s.info.fileIndex + let pathFi = if infoFi.int32 >= 0'i32: infoFi else: s.itemId.module.int32.FileIndex + c &= customPath(conf.toFullPath(pathFi)) when defined(icDbgHash): var ownerSteps = 0 while it != nil: diff --git a/compiler/transf.nim b/compiler/transf.nim index f131c862f0..f0cf9dd33b 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -1386,7 +1386,33 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flags: Transf result = getBody(g, prc) else: prc.transformedBody = newNode(nkEmpty) # protects from recursion - var c = openTransf(g, prc.getModule, "", idgen, flags) + # Lambda-lifting a routine body while the VM compiles it (to run a macro + # under `nim ic`) mints a closure `:env` (type + obj + fields + hidden param) + # that the lift welds into the routine's serialized signature. Such an env is + # a PROCESS-LOCAL artifact (its item number is per-process-sequential), so a + # reference to it must never carry a stable cross-module identity — otherwise + # a consumer resolves it against a canonical NIF built by a different process + # that has no matching def ('symbol has no offset', e.g. Nimbus t17.275). + # Lift in the backend (process-local) id space; ast2nif then emits these as + # module-local `@bk` defs (mirrors setAttachedOp's inVMTransform handling). + var liftIdgen = idgen + if g.inVMTransform > 0 and g.config.cmd == cmdM: + if g.vmTransfIdgen == nil: + g.vmTransfIdgen = idGeneratorForBackend(g.systemModule) + liftIdgen = g.vmTransfIdgen + var c = openTransf(g, prc.getModule, "", liftIdgen, flags) + # `liftCapturedVars` rewrites captured locals to `:env.field` IN PLACE on the + # body it is handed; the env-creation prologue lands only in the returned + # wrapper. When the VM drives this transform (running a macro/CT proc), that + # in-place mutation corrupts the routine's PRE-transform `ast[bodyPos]` — + # under IC exactly the node `getBody` serializes to the module's `.s.nif`. So + # snapshot the pristine body before the VM lift and restore `ast[bodyPos]` + # afterwards: the VM still consumes the fully-lifted `result`, but `getBody` + # keeps faithfully returning the pre-transform body for serialization. The + # cg/backend path (`inVMTransform == 0`) is untouched. + let vmPristineBody = + if g.inVMTransform > 0: copyTree(getBody(g, prc)) + else: nil result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen, flags) result = processTransf(c, result, prc) liftDefer(c, result) @@ -1396,6 +1422,8 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flags: Transf result = g.transformClosureIterator(c.idgen, prc, result) incl(result.flags, nfTransf) + if vmPristineBody != nil: + prc.ast[bodyPos] = vmPristineBody if useCache in flags or prc.typ.callConv == ccInline: # genProc for inline procs will be called multiple times from different modules, diff --git a/compiler/typekeys.nim b/compiler/typekeys.nim index 435b927e3a..d5e0b52e09 100644 --- a/compiler/typekeys.nim +++ b/compiler/typekeys.nim @@ -139,6 +139,25 @@ proc maybeImported(c: var Context; s: PSym; conf: ConfigRef) {.inline.} = if s != nil and {sfImportc, sfExportc} * s.flagsImpl != {}: c.symKey(s, conf) +proc backendTypeName(t: PType; conf: ConfigRef): string = + ## Stable cross-module identity of a backend-minted (lower-stage) type: its + ## serialized `@bk` NIF name (mirrors ast2nif.nifTypeName). A closure-env + ## object/ref minted by the `lower` stage has NO stable STRUCTURAL key — its + ## captured-field types re-resolve to different modules in the producing vs the + ## consuming process (e.g. field `x0` → `int` in the producer, → the consumer's + ## alias in the consumer) — but this name (kind + item + home-module suffix) is + ## identical in both, because the consumer loads the producer's name verbatim. + ## Keying hooks by it makes producer `setAttachedOp` and consumer `getAttachedOp` + ## agree. The trailing `@bk` (= ast2nif.BackendLocalMarker) keeps it disjoint + ## from any normal type's structural key. + result = "`t" + result.addInt ord(t.kind) + result.add '.' + result.addInt t.uniqueId.item + result.add '.' + result.add modname(t.uniqueId.module, conf) + result.add "@bk" + proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef) = if t == nil: c.m.addEmpty() @@ -148,6 +167,14 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef assert c.tl != nil c.tl(t) + if t.uniqueId.isBackendMinted: + # Backend-minted (lower-stage) closure-env types key by their stable NIF name, + # never by structure (which diverges across the NIF boundary). An env `ref` + # that is itself NOT backend-minted still keys stably: it recurses here and + # reaches its `@bk` object, which short-circuits to a stable name. + c.m.addSymbol backendTypeName(t, conf) + return + case t.kind of tyGenericInvocation: for a in t.sonsImpl: diff --git a/compiler/vm.nim b/compiler/vm.nim index 851ba1d9db..23c579dd17 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1313,8 +1313,41 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = # a macro observed this symbol's implementation: NeedsImpl edge to # its home module under IC. recordIcImplDep(c.graph, a.sym) - regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit) - else: copyTree(a.sym.ast) + if a.sym.ast.isNil: + regs[ra].node = newNode(nkNilLit) + else: + let tree = copyTree(a.sym.ast) + # A NIF-loaded routine's `ast[paramsPos]` is an `nkEmpty` placeholder: + # ast2nif strips the formal params (recoverable from `typ.n`, see + # writeNode's `skipParams`). A macro that reads `fn.getImpl[paramsPos]` + # — e.g. taskpools `spawn` reads the return type via `getImpl[3][0]` — + # needs them, so reconstruct a read-only formalParams from the proc + # type. The synthesized type-expression nodes carry the resolved + # `PType`, which is all a macro can query for a loaded routine. + if tree.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, + nkConverterDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo} and + tree.safeLen > paramsPos and tree[paramsPos].kind == nkEmpty and + a.sym.typ != nil and a.sym.typ.n != nil and + a.sym.typ.n.kind == nkFormalParams: + let t = a.sym.typ + let fp = newNodeI(nkFormalParams, a.sym.info) + let rt = t.returnType + # `opMapTypeInstToAst` (inst=true) reproduces a source-like type + # declaration — crucially it renders an array's range bound as + # `range 0..N` (the `inst=false` form emits `range[0, N]`, which + # re-sems to "'range' expects one type parameter"). + fp.add(if rt != nil: opMapTypeInstToAst(c.cache, rt, a.sym.info, c.idgen) + else: newNodeI(nkEmpty, a.sym.info)) + for i in 1 ..< t.n.len: + if t.n[i].kind == nkSym: + let p = t.n[i].sym + let def = newNodeI(nkIdentDefs, p.info) + def.add newIdentNode(p.name, p.info) + def.add opMapTypeInstToAst(c.cache, p.typ, p.info, c.idgen) + def.add newNodeI(nkEmpty, p.info) + fp.add def + tree[paramsPos] = fp + regs[ra].node = tree regs[ra].node.flags.incl nfIsRef else: stackTrace(c, tos, pc, "node is not a symbol") diff --git a/koch.nim b/koch.nim index a7333ba468..f1eac10910 100644 --- a/koch.nim +++ b/koch.nim @@ -618,7 +618,8 @@ proc runIcTestFile(inp: string) = # which exercises the NIF import/load path the single-file tests do not. const icSuite = ["thallo", "tconverter", "timp", "tmiscs", "tparseutils", "tcompiletimeglobal", "tsighashstable", "tpureenum", "tgenericoffer", - "tconverterreexport"] + "tconverterreexport", "ttypeoffer", "ttransitiveoffer", + "tmodsymref", "tmethupref"] proc icTest(args: string) = temp("") diff --git a/tests/ic/mmethupref.nim b/tests/ic/mmethupref.nim new file mode 100644 index 0000000000..862408b123 --- /dev/null +++ b/tests/ic/mmethupref.nim @@ -0,0 +1,23 @@ +# Helper for tmethupref: a `{.base.}` method whose body contains a closure +# iterator that in turn contains a nested closure capturing a method-local. +# The capture forces an `up` reference chain (nested closure -> iterator env -> +# method env). The method also gets a dispatcher (a `copySym` clone that shares +# the method's body sub-tree, including the iterator). Under `nim ic` the +# dispatcher used to be treated as an owned runtime routine and lambda-lifted in +# the per-module lower stage, lifting the SHARED iterator a second time under a +# different owner identity -> "up references do not agree" / "could not determine +# closure type". See nifbackend.ownsRuntimeRoutine (sfDispatcher exclusion). + +type Base* = ref object of RootObj + val*: int + +method compute*(b: Base): int {.base.} = + var acc = b.val + iterator steps(): int {.closure.} = + proc bump() = + acc += 1 + bump() + bump() + yield acc + for s in steps(): + result = s diff --git a/tests/ic/mmodsymadd.nim b/tests/ic/mmodsymadd.nim new file mode 100644 index 0000000000..75a5e3fe51 --- /dev/null +++ b/tests/ic/mmodsymadd.nim @@ -0,0 +1,12 @@ +import mmodsymasm + +# The dead `else` branch qualifies a re-exported module (`mmodsymarm`) whose +# `foo` is gated out on this host: the module sym binds at template-definition +# but the member never resolves, so a dangling module-symbol reference survives +# into the serialized template body. Before the `ModMarker` fix this failed +# under `nim ic` with `symbol has no offset` when the consumer loaded this NIF. +template satAdd*(a, b: uint64): uint64 = + when not defined(mmodSymFakeArch): + mmodsymasm.mmodsymx86.foo(a, b) + else: + mmodsymasm.mmodsymarm.foo(a, b) diff --git a/tests/ic/mmodsymarm.nim b/tests/ic/mmodsymarm.nim new file mode 100644 index 0000000000..c8d05dba82 --- /dev/null +++ b/tests/ic/mmodsymarm.nim @@ -0,0 +1,5 @@ +# Gated out on every host (mmodSymFakeArch is never defined): `foo` does NOT +# exist here, so a qualified `…mmodsymarm.foo` cannot resolve to the proc and +# leaves the bare re-exported MODULE symbol dangling in the template body. +when defined(mmodSymFakeArch): + func foo*(a, b: uint64): uint64 = a + b diff --git a/tests/ic/mmodsymasm.nim b/tests/ic/mmodsymasm.nim new file mode 100644 index 0000000000..ea5f73f30d --- /dev/null +++ b/tests/ic/mmodsymasm.nim @@ -0,0 +1,2 @@ +import mmodsymx86, mmodsymarm +export mmodsymx86, mmodsymarm diff --git a/tests/ic/mmodsymx86.nim b/tests/ic/mmodsymx86.nim new file mode 100644 index 0000000000..891568d49a --- /dev/null +++ b/tests/ic/mmodsymx86.nim @@ -0,0 +1,3 @@ +# Live on every real host: provides `foo` so the template's taken branch resolves. +when not defined(mmodSymFakeArch): + func foo*(a, b: uint64): uint64 = a + b diff --git a/tests/ic/mtoffcodec.nim b/tests/ic/mtoffcodec.nim new file mode 100644 index 0000000000..22037afdce --- /dev/null +++ b/tests/ic/mtoffcodec.nim @@ -0,0 +1,3 @@ +# Helper for ttypeoffer.nim: the extra overload that flips `compiles(toSszType(Gwei))`. +import mtoffgwei +template toSszType*(v: Gwei): uint64 = uint64(v) diff --git a/tests/ic/mtoffgwei.nim b/tests/ic/mtoffgwei.nim new file mode 100644 index 0000000000..04a395213a --- /dev/null +++ b/tests/ic/mtoffgwei.nim @@ -0,0 +1,2 @@ +# Helper for ttypeoffer.nim: a distinct basic type. +type Gwei* = distinct uint64 diff --git a/tests/ic/mtoffssz.nim b/tests/ic/mtoffssz.nim new file mode 100644 index 0000000000..6ee77cac94 --- /dev/null +++ b/tests/ic/mtoffssz.nim @@ -0,0 +1,10 @@ +# Helper for ttypeoffer.nim: a generic container whose hash-array bound depends +# on a `mixin toSszType` resolved at the instantiation site (cf. ssz dataPerChunk). +template perChunk*(T: type): int = + mixin toSszType + when compiles(toSszType(default(T))): 4 else: 1 + +type + HA*[N: static int; T] = object + data*: array[N, T] + hashes*: array[N div perChunk(T), uint64] diff --git a/tests/ic/mtoffstate.nim b/tests/ic/mtoffstate.nim new file mode 100644 index 0000000000..beba18e387 --- /dev/null +++ b/tests/ic/mtoffstate.nim @@ -0,0 +1,5 @@ +# Helper for ttypeoffer.nim ("datatypes" analog): instantiates HA[64,Gwei] WITHOUT +# the codec in scope -> perChunk=1. This is the instance that must be shared. +import mtoffssz, mtoffgwei +type StateA* = object + field*: HA[64, Gwei] diff --git a/tests/ic/mtoffuser.nim b/tests/ic/mtoffuser.nim new file mode 100644 index 0000000000..717fa7633d --- /dev/null +++ b/tests/ic/mtoffuser.nim @@ -0,0 +1,10 @@ +# Helper for ttypeoffer.nim ("db_immutable" analog): imports the codec (so +# `toSszType(Gwei)` is visible -> perChunk=4) and re-instantiates HA[64,Gwei]. +# With the `(toffer …)` fix it reuses mtoffstate's instance instead. +import mtoffssz, mtoffgwei, mtoffcodec, mtoffstate +type StateB* = object + field*: HA[64, Gwei] + +proc check*() = + static: doAssert sizeof(StateA) == sizeof(StateB) + echo "ok" diff --git a/tests/ic/mtscopea.nim b/tests/ic/mtscopea.nim new file mode 100644 index 0000000000..985a5da147 --- /dev/null +++ b/tests/ic/mtscopea.nim @@ -0,0 +1,2 @@ +# Helper for ttransitiveoffer.nim: const that the generic body binds at definition. +const TScopeSize* = 65 diff --git a/tests/ic/mtscopeb.nim b/tests/ic/mtscopeb.nim new file mode 100644 index 0000000000..8c119ca511 --- /dev/null +++ b/tests/ic/mtscopeb.nim @@ -0,0 +1,2 @@ +# Helper: a CONFLICTING const of the same name, visible only in the consumer. +const TScopeSize* = 33 diff --git a/tests/ic/mtscopegen.nim b/tests/ic/mtscopegen.nim new file mode 100644 index 0000000000..7e56d25b88 --- /dev/null +++ b/tests/ic/mtscopegen.nim @@ -0,0 +1,7 @@ +# Helper: defines a generic whose body uses TScopeSize via the strformat `&` +# macro (late-bound), resolved in THIS module's scope (mtscopea -> 65). +import mtscopea, std/strformat +export mtscopea +func fromRaw*[T](x: T): string = + const msg = &"size {TScopeSize - 1}" + result = msg & " " & $int(x) diff --git a/tests/ic/mtscopemid.nim b/tests/ic/mtscopemid.nim new file mode 100644 index 0000000000..84ec9ef11a --- /dev/null +++ b/tests/ic/mtscopemid.nim @@ -0,0 +1,3 @@ +# Helper: makes mtscopewarm a TRANSITIVE import of the consumer. +import mtscopewarm +export mtscopewarm diff --git a/tests/ic/mtscopewarm.nim b/tests/ic/mtscopewarm.nim new file mode 100644 index 0000000000..a70c3872e5 --- /dev/null +++ b/tests/ic/mtscopewarm.nim @@ -0,0 +1,4 @@ +# Helper: instantiates fromRaw[int] in a CLEAN scope (no mtscopeb) -> the +# correct instance that the consumer must reuse. +import mtscopegen +proc warm*(): string = fromRaw(5) diff --git a/tests/ic/tmethupref.nim b/tests/ic/tmethupref.nim new file mode 100644 index 0000000000..333138c7f1 --- /dev/null +++ b/tests/ic/tmethupref.nim @@ -0,0 +1,15 @@ +discard """ +output: '''42''' +""" + +# Regression test: a `{.base.}` method whose body holds a closure iterator with a +# nested capturing closure must not crash the IC backend. The method's dispatcher +# (a `copySym` clone sharing the iterator) must NOT be lambda-lifted per module; +# otherwise the shared iterator's `up` field is baked twice under divergent owner +# identities -> "up references do not agree" / "could not determine closure type" +# (the real-world symptom: ~all libp2p async `{.base.}` methods failed under +# `nim ic`). Fixed by excluding `sfDispatcher` from nifbackend.ownsRuntimeRoutine. + +import mmethupref +let b = Base(val: 40) +echo b.compute() diff --git a/tests/ic/tmodsymref.nim b/tests/ic/tmodsymref.nim new file mode 100644 index 0000000000..607f6da513 --- /dev/null +++ b/tests/ic/tmodsymref.nim @@ -0,0 +1,12 @@ +discard """ +output: '''7''' +""" + +# Regression test: a cross-module MODULE-symbol reference left as a dangling +# qualifier in a template body (here `mmodsymasm.mmodsymarm.foo` in a dead +# `when`-branch, reached via re-export) must load under `nim ic` instead of +# raising `symbol has no offset`. Mirrors nim-intops' `inlineasm.arm64.X` in +# nimbus-eth2. See compiler/ast2nif.nim `ModMarker`. + +import mmodsymadd +echo satAdd(3'u64, 4'u64) diff --git a/tests/ic/ttransitiveoffer.nim b/tests/ic/ttransitiveoffer.nim new file mode 100644 index 0000000000..1b14daa08d --- /dev/null +++ b/tests/ic/ttransitiveoffer.nim @@ -0,0 +1,14 @@ +discard """ +output: '''size 64 64''' +""" + +# Regression test for TRANSITIVE generic-instance offers. `fromRaw[int]` is first +# instantiated in mtscopewarm (a clean scope where `TScopeSize` is unambiguously +# 65). The consumer here also imports mtscopeb (`TScopeSize` = 33), so a fresh +# re-instantiation of fromRaw's body would resolve `TScopeSize` ambiguously. The +# clean instance reaches here only TRANSITIVELY (via mtscopemid), so the offer +# rebuild must walk the whole import closure, not just direct imports. Mirrors +# nimbus-eth2 `keys.fromRaw` -> `SkRawPublicKeySize` (secp vs secp256k1). +import mtscopegen, mtscopeb, mtscopemid + +echo fromRaw(64) diff --git a/tests/ic/ttypeoffer.nim b/tests/ic/ttypeoffer.nim new file mode 100644 index 0000000000..bdc25466b1 --- /dev/null +++ b/tests/ic/ttypeoffer.nim @@ -0,0 +1,14 @@ +discard """ +output: '''ok''' +""" + +# Regression test for the `(toffer …)` generic-TYPE-instance sharing across the +# NIF boundary. A `tyGenericInst` whose structure (here an `array` bound) depends +# on a `mixin`/`compiles()` resolved at the instantiation site must be REUSED +# from the module that created it, not re-instantiated in a consumer whose import +# scope flips the `compiles()` and so bakes a different bound. Mirrors the SSZ +# `HashArray[8192, Gwei]` `sizeof` divergence in nimbus-eth2. Before the fix this +# failed under `nim ic` with `doAssert sizeof(StateA) == sizeof(StateB)`. + +import mtoffuser +check()