# # # The Nim Compiler # (c) Copyright 2025 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## AST to NIF bridge. import std / [assertions, tables, sets] from std / strutils import startsWith, endsWith, contains from std / os import fileExists, dirExists, walkFiles, existsEnv, commandLineParams, getCurrentProcessId from std / exitprocs import addExitProc from std / syncio import readFile, stderr, writeLine from std / algorithm import sort import "../dist/checksums/src/checksums" / sha1 import astdef, idents, msgs, options import lineinfos as astli import pathutils #, modulegraphs import "../dist/nimony/src/lib" / [bitabs, nifstreams, lineinfos, nifindexes, nifreader] # Step 2b: the READER speaks nifcore; the WRITER keeps nifstreams (global `pool`, # PackedToken/PackedLineInfo). nifstreams does NOT export Cursor/TokenBuf/NifKind, # so those resolve unambiguously to nifcore. `except pool`: nifcore's # `pool(c: Cursor)` accessor would shadow nifstreams' global `pool` var the writer # uses; the reader reaches pools via `symName(c)`/`strVal(c)` etc. import "../dist/nimony/src/lib/nifcore" except pool from "../dist/nimony/src/lib" / bif import load, BifModule import icmodnames import "../dist/nimony/src/models" / nifindex_tags import typekeys import icnifcore import ic / [enum2nif] const SysModuleSuffix* = "@sys" const BackendLocalMarker* = "@bk" ## Suffix marker for a PROCESS-LOCAL backend-minted entity (a closure `:env` ## type/obj/field/hidden-param minted while the VM compiles a routine body to ## run a macro). Such entities have no stable cross-process identity, so each ## module that references one emits its OWN module-local def named ## `…@bk` and the loader homes it to the reading module with ## a `backendItemId` (disjoint from real ids). See transf.transformBody. ## Reserved module-suffix sentinel for module-less magic singleton types — the ## `nil` type is created via `newSysType` with the graph idgen, whose `module` ## can be `-1` (e.g. during VM const-eval before a real module is current), so ## its `uniqueId.module` is unresolvable. Such a type has no fields and an ## identity that is fully captured by its kind, so we serialize it with this ## sentinel and reconstruct it on load (see `createTypeStub`) without ever ## touching a `.nif` file. A real `moduleSuffix` never starts with '@'. proc typeToNifSym(typ: PType; config: ConfigRef): string = # NOTE: uniqueId is the serialization identity and is unique per instance — # `exactReplica` keeps only itemId shared with its original (see ast.nim) assert not typ.uniqueId.isBackendMinted result = "`t" result.addInt ord(typ.kind) result.add '.' result.addInt typ.uniqueId.item result.add '.' if typ.uniqueId.module < 0: result.add SysModuleSuffix else: result.add modname(typ.uniqueId.module, config) proc icNifTypeName*(typ: PType; config: ConfigRef): string = ## The serialized NIF name of a type, recorded next to RTTI data ## definitions in the cnif artifact so a later run can re-demand the ## typeinfo when a reused TU still references it (the def-retention ## check). Backend-minted types have no NIF name. if typ != nil and not typ.uniqueId.isBackendMinted: result = typeToNifSym(typ, config) else: result = "" proc toHookIndexEntry*(config: ConfigRef; typeId: ItemId; hookSym: PSym): HookIndexEntry = ## Converts a type ItemId and hook symbol to a HookIndexEntry for the NIF index. let typeSymName = "`t" & $typeId.item & "." & cachedModuleSuffix(config, typeId.module.FileIndex) let hookSymName = hookSym.name.s & "." & $hookSym.disamb & "." & cachedModuleSuffix(config, hookSym.itemId.module.FileIndex) let typSymId = pool.syms.getOrIncl(typeSymName) let hookSymId = pool.syms.getOrIncl(hookSymName) # Check if it's a generic hook (has non-empty generic params) let isGeneric = hookSym.astImpl != nil and hookSym.astImpl.len > genericParamsPos and hookSym.astImpl[genericParamsPos].kind != nkEmpty result = HookIndexEntry(typ: typSymId, hook: hookSymId, isGeneric: isGeneric) proc toConverterIndexEntry*(config: ConfigRef; converterSym: PSym): (nifstreams.SymId, nifstreams.SymId) = ## Converts a converter symbol to an index entry (destType, converterSym). ## Returns the destination type's SymId and the converter's SymId. # Get the return type of the converter (destination type) let retType = converterSym.typImpl if retType != nil and retType.sonsImpl.len > 0: let destType = retType.sonsImpl[0] # Return type is first son if destType != nil: let destTypeSymName = "`t" & $destType.itemId.item & "." & cachedModuleSuffix(config, destType.itemId.module.FileIndex) let convSymName = converterSym.name.s & "." & $converterSym.disamb & "." & cachedModuleSuffix(config, converterSym.itemId.module.FileIndex) result = (pool.syms.getOrIncl(destTypeSymName), pool.syms.getOrIncl(convSymName)) return # Fallback: return empty entry result = (nifstreams.SymId(0), nifstreams.SymId(0)) proc toMethodIndexEntry*(config: ConfigRef; methodSym: PSym; signature: string): (nifstreams.SymId, nifstreams.StrId) = ## Converts a method symbol/signature to a method index entry. let methodSymName = methodSym.name.s & "." & $methodSym.disamb & "." & cachedModuleSuffix(config, methodSym.itemId.module.FileIndex) result = ( pool.syms.getOrIncl(methodSymName), pool.strings.getOrIncl(signature) ) proc toClassSymId*(config: ConfigRef; typeId: ItemId): nifstreams.SymId = ## Converts a type ItemId to its SymId for the class index. let typeSymName = "`t" & $typeId.item & "." & cachedModuleSuffix(config, typeId.module.FileIndex) result = pool.syms.getOrIncl(typeSymName) # ---------------- Line info handling ----------------------------------------- type LineInfoWriter = object # `fileK`/`fileV` cache the most recently resolved (FileIndex -> FileId) pair, # faster than the hash table. `fileK` MUST be constructed at an invalid # sentinel (see `newLineInfoWriter`), never zero: `FileIndex(0)` is a real file # index, and `fileV` zero-inits to `FileId(0)` == `NoFile`, so a zero `fileK` # would make the first lookup of the module-at-index-0 falsely hit this cache # and return `NoFile` — silently dropping ALL of that module's line info. fileK: FileIndex fileV: FileId tab: Table[FileIndex, FileId] revTab: Table[FileId, FileIndex] # reverse mapping for oldLineInfo man: LineInfoManager config: ConfigRef proc newLineInfoWriter(config: ConfigRef): LineInfoWriter = # `fileK` starts invalid so the one-entry cache never collides with a real # `FileIndex(0)` (see the type's doc comment). LineInfoWriter(config: config, fileK: astli.InvalidFileIdx) proc get(w: var LineInfoWriter; key: FileIndex): FileId = if w.fileK == key: result = w.fileV else: if key in w.tab: result = w.tab[key] w.fileK = key w.fileV = result else: result = pool.files.getOrIncl(msgs.toFullPath(w.config, key)) w.fileK = key w.fileV = result w.tab[key] = result w.revTab[result] = key proc nifLineInfo(w: var LineInfoWriter; info: TLineInfo): PackedLineInfo = if info == unknownLineInfo: result = NoLineInfo else: let fid = get(w, info.fileIndex) # Must use pool.man since toString uses pool.man to unpack result = pack(pool.man, fid, info.line.int32, info.col) proc nifLineInfoWithComment(w: var LineInfoWriter; info: TLineInfo; doc: string): PackedLineInfo = ## Like `nifLineInfo` but also attaches `doc` as a NIF `#…#` comment on the ## token. Used to carry `##` doc comments, which the AST serialization itself ## drops, across a NIF round-trip (the loader reads it back off the info). if doc.len == 0: result = nifLineInfo(w, info) else: let cid = pool.strings.getOrIncl(doc).uint32 if info == unknownLineInfo: result = packWithComment(pool.man, NoFile, 0'i32, 0'i32, cid) else: let fid = get(w, info.fileIndex) result = packWithComment(pool.man, fid, info.line.int32, info.col, cid) proc oldLineInfo(w: var LineInfoWriter; info: NifLineInfo; p: Pool): TLineInfo = ## Step 2b: the reader's line info arrives as a nifcore `NifLineInfo`; resolve ## it to a `TLineInfo`. `info.file` indexes the loaded buffer's OWN filename ## pool `p` (= `cursorPool(n)`), which is the shared `icPool` for a text-parsed ## module but a fresh per-file pool for a `bif`-loaded one. if info.file == NoFile: result = unknownLineInfo else: let filePath = p.filenames[info.file] let fileIdx = msgs.fileInfoIdx(w.config, AbsoluteFile filePath) result = TLineInfo(line: info.line.uint16, col: info.col.int16, fileIndex: fileIdx) # ------------- Writer --------------------------------------------------------------- #[ Strategy: We produce NIF from the PNode structure as the single source of truth. NIF nodes can however, refer to PSym and PType, these get NIF names. If the PSym/PType belongs to the module that we are currently writing, we emit these fields as an inner NIF structure via the special tags `sd` and `td`. In fact it is only these tags that get the NIF `SymbolDef` kinds so that the lazy loading mechanism cannot be confused. We could also emit non-local symbols and types later as the index structure will tell us the precise offsets anyway. ]# const hiddenTypeTagName = "ht" symDefTagName = "sd" typeDefTagName = "td" var sdefTag = registerTag(symDefTagName) tdefTag = registerTag(typeDefTagName) hiddenTypeTag = registerTag(hiddenTypeTagName) type Writer = object deps: IcBuilder # include&import deps infos: LineInfoWriter currentModule: int32 decodedFileIndices: HashSet[FileIndex] locals: HashSet[ItemId] # track proc-local symbols inProc: int writtenTypes: seq[PType] # types sealed during a non-owning emit writtenSyms: seq[PSym] # reset afterwards so their owner can keep using them writtenPackages: HashSet[string] depSuffixes: HashSet[string] # module suffixes already emitted as `(import ...)` deps emittedBackendTypes: HashSet[(int32, int32)] # backend-local types already def'd this # module, keyed by (kind, item): the NIF name is `t..@bk`, # so two `@bk` types sharing an item but differing in kind (e.g. `int` # and `typedesc[int]`, both item 12) are DISTINCT defs — keying by item # alone deduped the second to a dangling `SymUse` (`symbol has no offset`). emittedBackendSyms: HashSet[int32] # backend-local sym items already def'd this module lowering: bool # serializing the `lower` stage's whole-module `.t.nif` emittedFieldSyms: HashSet[ItemId] # lowering: derived env-field syms already def'd inTypeReclist: int # >0 while writing a type's OWN reclist: fields must be SELF-CONTAINED # defs (the type can be seek-loaded in isolation), not entry-deduped uses proc isLocalSym(sym: PSym): bool {.inline.} = ## Every symbol is emitted as a *global* (module-suffixed) name so that its ## `sdef` gets an index entry and is resolvable by index lookup even when ## referenced from a different index entry than the one that physically ## contains the definition. This matters for symbols shared across entries: ## generic params of a forward declaration vs its implementation, and proc-type ## params shared between an enclosing proc and a nested object's proc-type ## field. The per-module `disamb` counter keeps `name.disamb.module` unique, so ## globalising cannot cause clashes. This trades index size for correctness; ## size/speed can be optimised later. false const FieldMarker = "`f" ## Appended to the ident of `skField` symbols in NIF names. Object fields are ## emitted as *local* symbols (NIF spec sense): ``f.` with NO ## module suffix, so they get no index entry and are never registered in the ## global `c.syms` name table. A field reference is a leaf — its C member name ## is a deterministic function of `name.s` (`ccgtypes.mangleField`) and is ## struct-scoped, and the field's type already rides on the `PNode` — so there ## is nothing to resolve across modules: the use site just stubs a `skField` ## from the local name. This removes the whole foreign-suffix pollution class ## (a derived/captured env field minted under a foreign module suffix used to ## corrupt the loader's name→buffer seek). The `` `f `` marker keeps the field's ## local name in a namespace disjoint from proc-locals (backtick cannot appear ## in a Nim identifier), so a field use can never be misrouted to a same-named ## local var/param. Mirrors the `` `t `` (`typeToNifSym`) and `PkgMarker` ## namespaces. PkgMarker = "`pkg" ## Appended to the ident of `skPackage` symbols in NIF names. A package sym ## has no module of its own: it is written once into every module NIF that ## references it, named with that module's suffix and its own (independent) ## disamb counter. Without the marker it can collide with a module-level ## symbol of the same name and disamb — e.g. extccomp's `compiler` template ## vs the `compiler` package — and the module sym's owner then resolves to ## the wrong symbol on load, producing a cyclic owner chain that hangs every ## owner-walk (sighashes.hashSym etc.). Backtick cannot appear in a Nim ## identifier, mirroring the "`t" namespace used by `typeToNifSym`. proc toNifSymName(w: var Writer; sym: PSym): string = ## Generate NIF name for a symbol: local names are `ident.disamb`, ## global names are `ident.disamb.moduleSuffix` if sym.kindImpl == skField: # Object fields are LOCAL symbols (no module suffix, no index entry, not in the # global `c.syms`). See `FieldMarker`. The same `toNifSymName` call produces this # name at both the reclist def site and every use site (same `PSym`), so they # agree by construction; the loader recovers `name.s` and `mangleField` produces # the matching struct member name regardless of which module references it. result = sym.name.s result.add FieldMarker result.add '.' # Use the field's POSITION as the local name's numeric component: it is unique # within the owning type (so the local name is unambiguous there) AND it is what # tuple element access reads off a use-site field stub (`genRecordField`'s # tyTuple branch emits `Field$position`). Named-object field uses re-navigate by # name, so position is only load-bearing for tuples — but carrying it is free. result.addInt sym.positionImpl return if sym.itemId.isBackendMinted: # Process-local backend sym (closure env field / hidden `:env` param minted # during a VM transform): re-home to the current module with the `@bk` # marker so each referencing module self-contains it. See transformBody. # # Use `itemId.item` (the writer's dedup identity, see `emittedBackendSyms`) # as the numeric name component, NOT `disamb`: closure `:env` syms in one # module are minted from TWO id spaces — the backend lower stage's # `tb.idgen` and sem's `vmTransfIdgen` (transf.transformBody) — whose # `disambTable`s each start `:env` at the same low count, so a macro-lowered # `:env` (e.g. `implementSendProcBody`) and a backend-lowered one # (`peerTrimmerHeartbeat`) collide on `:env.2.@bk`. Two distinct syms # then share a NIF name; the loader's name-keyed index/`c.syms` return the # first for both, so one proc's `:env` gets the OTHER proc's env type # (mismatched-pointer C, "has no member colonup_" at link). `itemId.item` is # unique per `@bk` sym (both are emitted as defs, see writeSym), mirroring # how `@bk` TYPES already key off `uniqueId.item` (nifTypeName). The loader # copies this back into `disamb` (sn.count), so `globalName` round-trips. result = sym.name.s result.add '.' result.addInt sym.itemId.item result.add '.' result.add modname(w.currentModule, w.infos.config) result.add BackendLocalMarker return result = sym.name.s if sym.kindImpl == skPackage: result.add PkgMarker result.add '.' result.addInt sym.disamb if not isLocalSym(sym) and sym.itemId notin w.locals: # Global symbol: ident.disamb.moduleSuffix result.add '.' let module = if sym.kindImpl == skPackage: w.currentModule else: sym.itemId.module result.add modname(module, w.infos.config) proc globalName*(sym: PSym; config: ConfigRef): string = result = sym.name.s if sym.kindImpl == skPackage: # stubs store the clean name; the NIF index is keyed by the marked one result.add PkgMarker result.add '.' result.addInt sym.disamb result.add '.' result.add modname(sym.itemId.module, config) # A loaded process-local backend sym keeps its `@bk` marker in the NIF name # (the index/`c.syms` tables are keyed by it); mirror toNifSymName so name-based # lookups via globalName don't miss (KeyError `:env.N.` without the marker). if sym.itemId.isBackendMinted: result.add BackendLocalMarker type ParsedSymName* = object name*: string module*: string count*: int proc parseSymName*(s: string): ParsedSymName = var i = s.len - 2 while i > 0: if s[i] == '.': if s[i+1] in {'0'..'9'}: var count = ord(s[i+1]) - ord('0') var j = i+2 while j < s.len and s[j] in {'0'..'9'}: count = count * 10 + ord(s[j]) - ord('0') inc j return ParsedSymName(name: substr(s, 0, i-1), module: "", count: count) else: let mend = s.high var b = i-1 while b > 0 and s[b] != '.': dec b var j = b+1 var count = 0 while j < s.len and s[j] in {'0'..'9'}: count = count * 10 + ord(s[j]) - ord('0') inc j return ParsedSymName(name: substr(s, 0, b-1), module: substr(s, i+1, mend), count: count) dec i return ParsedSymName(name: s, module: "") proc isFieldNifName(name: string): bool {.inline.} = ## True for an object field's local NIF name ``f.` (see ## `FieldMarker`): no module suffix, marker on the ident. let sn = parseSymName(name) sn.module.len == 0 and sn.name.endsWith(FieldMarker) proc stubKindAndName(cache: IdentCache; rawName: string): (TSymKind, PIdent) = ## The user-visible name of a symbol stub must NOT keep NIF-only name ## decorations: the `PkgMarker` of package symbols would otherwise leak into ## every reader of `name.s` that runs before the stub is fully loaded ## (e.g. vmgen's callback keys built from owner chains). The marker also ## tells us the symbol kind up front, which `globalName` uses to rebuild ## the marked NIF name for the index lookup. if rawName.endsWith(PkgMarker): (skPackage, cache.getIdent(rawName[0 ..< rawName.len - PkgMarker.len])) elif rawName.endsWith(FieldMarker): # Object field (local NIF symbol, see `FieldMarker`): strip the marker so the # backend mangles the clean field name, and record the kind so a use-site stub # is a real `skField` (cgen branches on it for `obj.field` access). (skField, cache.getIdent(rawName[0 ..< rawName.len - FieldMarker.len])) else: (skStub, cache.getIdent(rawName)) # --- nifcore writer adapter ------------------------------------------------- # The `write*` procs build the module into an `IcBuilder` (nifcore). These give # the IcBuilder the SAME call shapes the old `nifstreams` TokenBuf had (taking # `nifstreams` TagId/SymId/PackedToken + a PackedLineInfo), so the writer bodies # are unchanged apart from the `var TokenBuf` -> `var IcBuilder` parameter type. # Line info is unpacked from the PackedLineInfo and re-emitted via # `IcBuilder.lineInfo` (absolute file/line/col; nifcore makes it relative on # serialization), exactly as the deleted `serializeViaNifcore` bridge did. proc emitInfo(b: var IcBuilder; info: PackedLineInfo) {.inline.} = if info.isValid: let u = unpack(pool.man, info) let fname = if u.file.isValid: pool.files[u.file] else: "" let cstr = if u.comment != 0'u32: pool.strings[nifstreams.StrId(u.comment)] else: "" b.lineInfo(fname, u.line, u.col, cstr) proc addParLe(b: var IcBuilder; tag: nifstreams.TagId; info = NoLineInfo) = b.openTag(pool.tags[tag]); b.emitInfo(info) proc addParRi(b: var IcBuilder) {.inline.} = b.closeTag() proc addSymDef(b: var IcBuilder; s: nifstreams.SymId; info = NoLineInfo) = b.addSymDef(pool.syms[s]); b.emitInfo(info) proc addSymUse(b: var IcBuilder; s: nifstreams.SymId; info = NoLineInfo) = b.addSymUse(pool.syms[s]); b.emitInfo(info) proc add(b: var IcBuilder; t: PackedToken) = ## Bridge a single old-API token constructor (symToken/strToken/floatToken/ ## charToken) into the IcBuilder. case t.kind of DotToken: b.addDotToken() of Ident: b.addIdent(pool.strings[t.litId]) of Symbol: b.addSymUse(pool.syms[t.symId]) of SymbolDef: b.addSymDef(pool.syms[t.symId]) of IntLit: b.addIntLit(pool.integers[t.intId]) of UIntLit: b.addUIntLit(pool.uintegers[t.uintId]) of FloatLit: b.addFloatLit(pool.floats[t.floatId]) of CharLit: b.addCharLit(char(t.uoperand)) of StringLit: b.addStrLit(pool.strings[t.litId]) else: discard if t.kind != ParRi: b.emitInfo(t.info) template buildTree(dest: var IcBuilder; tag: nifstreams.TagId; body: untyped) = dest.addParLe tag body dest.addParRi template buildTree(dest: var IcBuilder; tag: string; body: untyped) = dest.openTag tag body dest.closeTag() template buildTree(dest: var IcBuilder; tag: nifstreams.TagId; info: PackedLineInfo; body: untyped) = dest.addParLe tag, info body dest.addParRi proc writeFlags[E](dest: var IcBuilder; flags: set[E]) = var flagsAsIdent = "" genFlags(flags, flagsAsIdent) if flagsAsIdent.len > 0: dest.addIdent flagsAsIdent else: dest.addDotToken proc trLineInfo(w: var Writer; info: TLineInfo): PackedLineInfo {.inline.} = result = nifLineInfo(w.infos, info) proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) proc writeType(w: var Writer; dest: var IcBuilder; typ: PType) proc writeSym(w: var Writer; dest: var IcBuilder; sym: PSym) func restoresWrittenState(config: ConfigRef): bool {.inline.} = config.ideActive or optGenBif in config.globalOptions proc writeLoc(w: var Writer; dest: var IcBuilder; loc: TLoc) = dest.addIdent toNifTag(loc.k) dest.addIdent toNifTag(loc.storage) writeFlags(dest, loc.flags) # TLocFlags dest.addStrLit loc.snippet proc nifTypeName(w: Writer; typ: PType): string = ## NIF name of a type as written by THIS module. A process-local backend env ## type is re-homed to the current module with the `@bk` marker (see ## BackendLocalMarker); everything else uses the canonical `typeToNifSym`. if typ.uniqueId.isBackendMinted: result = "`t" result.addInt ord(typ.kind) result.add '.' result.addInt typ.uniqueId.item result.add '.' result.add modname(w.currentModule, w.infos.config) result.add BackendLocalMarker else: result = typeToNifSym(typ, w.infos.config) proc writeTypeDef(w: var Writer; dest: var IcBuilder; typ: PType) = dest.buildTree tdefTag: dest.addSymDef pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo dest.addDotToken # always private for the index generator #dest.addIdent toNifTag(typ.kind) writeFlags(dest, typ.flagsImpl) dest.addIdent toNifTag(typ.callConvImpl) dest.addIntLit typ.sizeImpl dest.addIntLit typ.alignImpl dest.addIntLit typ.paddingAtEndImpl dest.addIntLit typ.itemId.item # nonUniqueId # `exactReplica` keeps the canonical type's itemId (binding-table key) # while minting a fresh uniqueId (the NIF name): when the two halves # name different modules, the loader cannot reconstruct itemId.module # from the type's name — serialize it explicitly if typ.itemId.module != typ.uniqueId.module and not typ.itemId.isBackendMinted: dest.addStrLit modname(typ.itemId.module, w.infos.config) else: dest.addDotToken writeType(w, dest, typ.typeInstImpl) #if typ.kind in {tyProc, tyIterator} and typ.nImpl != nil and typ.nImpl.kind != nkFormalParams: # The reclist holds this type's OWN fields. A type can be force-loaded by # name in isolation (cg seeks the `.t.nif`/`.s.nif` index entry), so its # fields must be DEFS here, not entry-deduped SymUses whose def lives # elsewhere in the `(lowered)` entry and is never read by the seek. # # `emittedFieldSyms` only guards against a field being def'd twice WITHIN one # reclist, so scope it per-reclist: a generic object and its instances SHARE one # field PSym (same itemId) yet each instance carries a DISTINCT field type (e.g. # `MDigest[256].data: array[32,byte]` vs `MDigest[384].data: array[48,byte]`), so # each reclist needs its OWN typed def. A Writer-global set deduped every instance # after the first to a typeless `SymUse` stub (nil typ/owner on load → crash in # destructor lifting). Field NIF names are local (no module suffix, not in the # global `c.syms`), so def'ing the same field in two reclists never collides. inc w.inTypeReclist let savedFieldSyms = move w.emittedFieldSyms writeNode(w, dest, typ.nImpl) w.emittedFieldSyms = savedFieldSyms dec w.inTypeReclist writeSym(w, dest, typ.ownerFieldImpl) writeSym(w, dest, typ.symImpl) # Write TLoc structure writeLoc w, dest, typ.locImpl # we store the type's elements here at the end so that # it is not ambiguous and saves space: for ch in typ.sonsImpl: writeType(w, dest, ch) proc writeType(w: var Writer; dest: var IcBuilder; typ: PType) = if typ == nil: dest.addDotToken() elif typ.uniqueId.isBackendMinted: # Process-local closure env (see transf.transformBody): emit a MODULE-LOCAL # `@bk` def the first time it is reached in this module, reference it after. # Per-Writer dedup (NOT the shared `state`), since every referencing module # must emit its own copy. if not w.emittedBackendTypes.containsOrIncl((ord(typ.kind).int32, typ.uniqueId.item)): writeTypeDef(w, dest, typ) else: dest.addSymUse pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo elif typ.uniqueId.module == w.currentModule and typ.state == Complete: # Ownership for serialization is decided by `uniqueId`, not `itemId`: the NIF # name (`typeToNifSym`) and the loader (`createTypeStub`) both key off # `uniqueId`, so the module that *created* the type (uniqueId.module) must be # the one that emits its definition. `itemId.module` can be reassigned and # diverge from `uniqueId.module`; gating on it filed the def in the wrong # module (or nowhere), leaving dangling references (e.g. `symbol has no # offset` for a `pointer` type whose itemId.module drifted away). typ.state = Sealed if restoresWrittenState(w.infos.config): w.writtenTypes.add typ writeTypeDef(w, dest, typ) else: dest.addSymUse pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo proc writeBool(dest: var IcBuilder; b: bool) = dest.buildTree (if b: "true" else: "false"): discard proc writeLib(w: var Writer; dest: var IcBuilder; lib: PLib) = if lib == nil: dest.addDotToken() else: dest.buildTree toNifTag(lib.kind): dest.writeBool lib.generated dest.writeBool lib.isOverridden dest.addStrLit lib.name writeNode w, dest, lib.path proc docOfSym(sym: PSym): string = ## The `##` doc comment documenting `sym`, if any (mirrors nifler's ## docCommentOf). The comment may sit on the decl node itself or as the first ## `nkCommentStmt` of a routine body. Carried separately on the sym def's NIF ## token because the AST serialization drops comments — nimsuggest needs it ## for "find definition" doc hovers. let n = sym.astImpl if n == nil or nodeCommentReader == nil: return "" let own = nodeCommentReader(n) if own.len > 0: return own if sym.kindImpl in routineKinds and n.safeLen > bodyPos: let body = n[bodyPos] if body != nil and body.kind == nkStmtList and body.len > 0 and body[0].kind == nkCommentStmt: return nodeCommentReader(body[0]) return "" proc writeSymDef(w: var Writer; dest: var IcBuilder; sym: PSym) = dest.addParLe sdefTag, nifLineInfoWithComment(w.infos, sym.infoImpl, docOfSym(sym)) dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo # The `x` marker means "importable as a bare identifier into an importer's # scope". Object fields carry `sfExported` (so they are visible via `obj.field` # across modules) but must NOT become bare-importable: otherwise an exported # field name (e.g. `HSlice.a`, whose type is a generic param `T`) leaks into # module scope and a template's open/mixin symbol of the same name resolves to # the field instead of a local, producing "type mismatch: got 'T'". Fields are # still indexed (for `obj.field` resolution via the loaded object type); they # are merely not advertised as importable. Plain `skEnumField` stays importable # — enum values are legitimately usable as bare identifiers — but a field of a # `{.pure.}` enum is NOT: the source path keeps pure fields out of the importer # scope (`declarePureEnumField`), reachable only qualified or via the restricted # pure-enum mechanism (`importPureEnumFields`, fed by `ifaces[].pureEnums` which # a loaded module rebuilds from its `PureEnumEntry` log ops). Marking them # bare-importable made a loaded pure enum's fields leak into module scope # (`populateInterfaceTablesFromIndex` adds every `x`/Exported sym to `interf`), # e.g. nim-json-serialization's pure `JsonValueKind.Number` shadowing web3's # `Number = distinct uint64` so `uint64(x).Number` failed under `nim ic` # ("undeclared field 'Number'"). let isPureEnumField = sym.kindImpl == skEnumField and sym.typImpl != nil and sym.typImpl.symImpl != nil and sfPure in sym.typImpl.symImpl.flagsImpl if sym.kindImpl != skField and not isPureEnumField and {sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}: dest.addIdent "x" else: dest.addDotToken # field `disamb` made part of the name, so do not store it here dest.buildTree sym.kindImpl.toNifTag: case sym.kindImpl of skLet, skVar, skField, skForVar: writeSym(w, dest, sym.guardImpl) dest.addIntLit sym.bitsizeImpl dest.addIntLit sym.alignmentImpl else: discard if sym.magicImpl == mNone: dest.addDotToken else: dest.addIdent toNifTag(sym.magicImpl) writeFlags(dest, sym.flagsImpl) writeFlags(dest, sym.optionsImpl) dest.addIntLit sym.offsetImpl if sym.kindImpl == skModule: dest.addDotToken() # position will be set by the loader! elif sym.kindImpl in {skVar, skLet, skForVar, skResult}: dest.addIntLit 0 # hack for the VM which uses this field to store information else: dest.addIntLit sym.positionImpl writeLib(w, dest, sym.annexImpl) # Generic params are written as *global* symbols (with a module suffix) so that # they get their own index entries and can be looked up lazily. This matters for # generic routines that have a separate forward declaration and implementation: # the two share the same generic param symbols, but each is serialized as its own # index entry. If the params were local, a reference from the implementation's # entry could not resolve the sdef emitted in the forward declaration's entry. writeType(w, dest, sym.typImpl) writeSym(w, dest, sym.ownerFieldImpl) # Store the AST for routine symbols and constants # Constants need their AST for astdef() to return the constant's value writeNode(w, dest, sym.astImpl, forAst = true) writeLoc w, dest, sym.locImpl writeNode(w, dest, sym.constraintImpl) writeSym(w, dest, sym.instantiatedFromImpl) # The TRANSFORMED body (ic_ideas.md 2-way body): a routine run at compile time # (macro / VM transform / `static`) already has its lowered body — closure # `:env` and all — computed during sem; serialize it so the backend reuses it # instead of re-deriving (the divergence behind the t17.275 env class). An # empty `.` here means "same as the semchecked body OR to be found in the # `.t.nif`" (the `lower` stage fills that gap). Non-routines / not-yet- # transformed routines write the empty marker. (`transformedBodyImpl` only # exists in the routine branch of the `TSym` variant.) if sym.kindImpl in routineKinds: writeNode(w, dest, sym.transformedBodyImpl) else: dest.addDotToken dest.addParRi proc shouldWriteSymDef(w: var Writer; sym: PSym): bool {.inline.} = # Don't write module/package symbols - they don't have NIF files if sym.kindImpl == skPackage: return not w.writtenPackages.containsOrIncl(sym.name.s) # Already written - don't write again if sym.state == Sealed: return false # If the symbol belongs to current module and would be written WITHOUT module suffix # (due to being in w.locals or being in skLocalSymKinds), it MUST have an sdef. # Otherwise it gets written as a bare SymUse and can't be found when loading. if sym.itemId.module == w.currentModule: if sym.itemId in w.locals or isLocalSym(sym): return true # Would be written without module suffix, needs sdef if sym.state == Complete: return true # Normal case for global symbols return false proc fieldDefHere(w: var Writer; sym: PSym): bool {.inline.} = ## An object field is a LOCAL symbol (see `FieldMarker`): it is DEF'd exactly once ## — inline in its owning type's reclist — and referenced as a bare `SymUse` ## everywhere else (resolved by the consumer re-navigating the object type by ## name; nothing else to recover). So write a def iff we are inside that reclist ## (`inTypeReclist > 0`); `emittedFieldSyms` guards against a field appearing ## twice in one reclist (e.g. a discriminant). Each type's reclist is thus ## self-contained, which is what a seek-load of a single `.t.bif` type entry needs. sym.kindImpl == skField and w.inTypeReclist > 0 and not w.emittedFieldSyms.containsOrIncl(sym.itemId) proc writeSym(w: var Writer; dest: var IcBuilder; sym: PSym) = if sym == nil: dest.addDotToken() elif sym.kindImpl == skField: if fieldDefHere(w, sym): writeSymDef(w, dest, sym) else: dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo elif sym.itemId.isBackendMinted: # Process-local backend sym (closure env field / hidden `:env` param): emit a # MODULE-LOCAL `@bk` def the first time, reference it after. Per-Writer dedup. if not w.emittedBackendSyms.containsOrIncl(sym.itemId.item): writeSymDef(w, dest, sym) else: dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo elif shouldWriteSymDef(w, sym): sym.state = Sealed if restoresWrittenState(w.infos.config): w.writtenSyms.add sym writeSymDef(w, dest, sym) else: # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo proc writeSymNode(w: var Writer; dest: var IcBuilder; n: PNode; sym: PSym) = if sym == nil: dest.addDotToken() return # Compare lazy-aware, not the raw field: a sym node loaded from a NIF carries # `typField == nil` plus `nfLazyType`, meaning "my type is the symbol's # type". Comparing `typField` directly would re-serialize such a node as # `(ht . sym)` — an explicitly nil node type — and the next loader gets a # nil-typed node *without* the lazy fallback (semfold & friends crash on # `n.typ == nil`). Only a genuinely nil node type keeps the explicit form. # (ast.nim's `typ` accessor is not importable here; replicate its fallback. # For a still-Partial sym `typImpl` is nil, which also compares equal below # and yields the plain SymUse form — exactly the lazy round-trip we want.) var nodeTyp = n.typField if nodeTyp == nil and nfLazyType in n.flags: nodeTyp = sym.typImpl # Backend-minted syms (process-local closure `:env` param/fields) are emitted # as MODULE-LOCAL `@bk` defs the first time reached this module (per-Writer # dedup shared with `writeSym`), regardless of module: their itemId.module is # the systemModule of `vmTransfIdgen`, so `shouldWriteSymDef` (which gates on # currentModule) would otherwise only ever emit a SymUse → dangling def. let isField = sym.kindImpl == skField let wantDef = if isField: fieldDefHere(w, sym) # def only inside the owning reclist (see fieldDefHere) elif sym.itemId.isBackendMinted: not w.emittedBackendSyms.containsOrIncl(sym.itemId.item) else: shouldWriteSymDef(w, sym) if wantDef: if not sym.itemId.isBackendMinted and not isField: sym.state = Sealed if restoresWrittenState(w.infos.config): w.writtenSyms.add sym if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): writeType(w, dest, nodeTyp) writeSymDef(w, dest, sym) else: writeSymDef(w, dest, sym) else: # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! let info = trLineInfo(w, n.info) # A field SymUse is a typeless leaf stub on load (its def lives in another seek), # so it cannot supply a lazy type — carry its type EXPLICITLY via the hidden-type # wrapper. `genFieldObjConstr`/object-init read `nField.typ` directly, so the node # must keep it. A field-use node often has a nil node-type (the type lives on the # sym), so fall back to the field sym's own type. if isField: let fieldTyp = if nodeTyp != nil: nodeTyp else: sym.typImpl dest.buildTree hiddenTypeTag, info: writeType(w, dest, fieldTyp) dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info elif nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, info: writeType(w, dest, nodeTyp) dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info else: dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info proc writeNodeFlags(dest: var IcBuilder; flags: set[TNodeFlag]) {.inline.} = writeFlags(dest, flags) template withNode(w: var Writer; dest: var IcBuilder; n: PNode; body: untyped) = dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) writeNodeFlags(dest, n.flags) writeType(w, dest, n.typField) body dest.addParRi proc addLocalSym(w: var Writer; n: PNode) = ## Previously forced proc-local symbols to be written without a module suffix. ## All symbols are now emitted as global (see `isLocalSym`), so `w.locals` is ## intentionally left empty. discard proc addLocalSyms(w: var Writer; n: PNode) = case n.kind of nkIdentDefs, nkVarTuple: # nkIdentDefs: [ident1, ident2, ..., type, default] # All children except the last two are identifiers for i in 0 ..< max(0, n.len - 2): addLocalSyms(w, n[i]) of nkPostfix: addLocalSyms(w, n[1]) of nkPragmaExpr: addLocalSyms(w, n[0]) of nkSym: addLocalSym(w, n) else: discard proc trInclude(w: var Writer; n: PNode) = w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) w.deps.addDotToken # flags w.deps.addDotToken # type for child in n: assert child.kind == nkStrLit w.deps.addStrLit child.strVal # raw string literal, no wrapper needed w.deps.addParRi proc moduleSuffix(conf: ConfigRef; f: FileIndex): string = cachedModuleSuffix(conf, f) proc trImport(w: var Writer; n: PNode) = for child in n: if child.kind == nkSym and child.sym.kindImpl == skModule: # a non-module sym appears for an `import v` inside an unexpanded # template body (e.g. stew/importops' `when compiles((; import v))`): # not a dependency edge, the import resolves at the expansion site w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) w.deps.addDotToken # flags w.deps.addDotToken # type let s = child.sym let fp = moduleSuffix(w.infos.config, s.positionImpl.FileIndex) w.deps.addStrLit fp # raw string literal, no wrapper needed w.deps.addParRi w.depSuffixes.incl fp proc trExport(w: var Writer; n: PNode) = # Collect export information for the index # nkExportStmt children are nkSym nodes # When exporting a module (export dollars), the module symbol is a child # followed by all symbols from that module - we use empty set to mean "export all" # When exporting specific symbols (export foo, bar), we collect their names w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) w.deps.addDotToken # flags w.deps.addDotToken # type for child in n: if child.kind == nkSym: let s = child.sym if s.kindImpl == skModule: discard "do not write module syms here" else: w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(s)), NoLineInfo w.deps.addParRi var replayTag = registerTag("replay") var repConverterTag = registerTag("repconverter") var repDestroyTag = registerTag("repdestroy") var repWasMovedTag = registerTag("repwasmoved") var repCopyTag = registerTag("repcopy") var repSinkTag = registerTag("repsink") var repDupTag = registerTag("repdup") var repTraceTag = registerTag("reptrace") var repDeepCopyTag = registerTag("repdeepcopy") var repEnumToStrTag = registerTag("repenumtostr") var repMethodTag = registerTag("repmethod") var repPureEnumTag = registerTag("reppureenum") #var repClassTag = registerTag("repclass") var includeTag = registerTag("include") var importTag = registerTag("import") var implTag = registerTag("implementation") var reexpModTag = registerTag("reexpmod") var offerTag = registerTag("offer") var typeOfferTag = registerTag("toffer") var modulesrcTag = registerTag("modulesrc") var expansionTag = registerTag("expansion") # `(sig )*` — signature occurrences (parameter names and the symbols # in their type expressions). A semchecked routine's params are dropped from the # serialized AST (`skipParams`) and reconstructed from `s.typ`, which holds the # RESOLVED type — so the source parameter names and the written type names (e.g. # an alias `Stream`, not `StreamObj`) carry no position in the module body. Like # the `expansion` records, these are teed into the `deps` side-channel: the loader # skips the tag, but `idetools` scans every Symbol token, so goto-def / find-usages # work on signatures. var sigTag = registerTag("sig") # `(unusedid )` — the module's first FREE itemId after the frontend # (`.s.bif`) or the lower stage (`.t.bif`). The backend seeds its per-module # sym/type counters here so freshly-minted backend ids (closure envs, RTTI # hooks, temps) start ABOVE every loaded id — no `toId` collision is possible # by construction (replaces relying on the `@bk` module-marker bit, which the # loader dropped on type USES). Mirrors NIF's `.unusedname` directive. var unusedIdTag = registerTag("unusedid") proc registerNifAstTags*() = ## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag` ## initializers above depend on `nifstreams.pool` having been initialized ## FIRST (`pool = createLiterals(TagData)` in nifstreams' module init) — an ## inter-module init-order requirement. The IC-built compiler currently emits ## module init calls in a different order, so the initializers registered ## into a pool that was subsequently replaced: the tag ids then denoted ## builtin tags (`replay` came out as `deref`, `repdestroy` as `pat`, ...) ## and every written NIF was silently corrupted. Called from `nim.nim` ## before any command runs; idempotent (`getOrIncl` by name). sdefTag = registerTag(symDefTagName) tdefTag = registerTag(typeDefTagName) hiddenTypeTag = registerTag(hiddenTypeTagName) replayTag = registerTag("replay") repConverterTag = registerTag("repconverter") repDestroyTag = registerTag("repdestroy") repWasMovedTag = registerTag("repwasmoved") repCopyTag = registerTag("repcopy") repSinkTag = registerTag("repsink") repDupTag = registerTag("repdup") repTraceTag = registerTag("reptrace") repDeepCopyTag = registerTag("repdeepcopy") repEnumToStrTag = registerTag("repenumtostr") repMethodTag = registerTag("repmethod") repPureEnumTag = registerTag("reppureenum") includeTag = registerTag("include") importTag = registerTag("import") implTag = registerTag("implementation") reexpModTag = registerTag("reexpmod") offerTag = registerTag("offer") typeOfferTag = registerTag("toffer") modulesrcTag = registerTag("modulesrc") expansionTag = registerTag("expansion") sigTag = registerTag("sig") proc emitSigOccurrences(w: var Writer; n: PNode) = ## Record every `nkSym` in a routine-signature subtree (parameter names and the ## symbols inside their type expressions, incl. the return type) as a `(sig ...)` ## occurrence in the `deps` side-channel, carrying the SOURCE position. Called on ## the params AST that `skipParams` is about to drop, so tooling keeps a ## positioned token for each signature symbol without changing the module body ## the loader / backend actually consume. if n == nil: return if n.kind == nkSym: w.deps.addParLe sigTag, NoLineInfo w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(n.sym)), trLineInfo(w, n.info) w.deps.addParRi else: for i in 0 ..< n.safeLen: emitSigOccurrences(w, n[i]) proc emitFwdDecl(w: var Writer; n: PNode; sym: PSym) = ## A routine's forward declaration (`proc foo(...)` with no body, later followed ## by `proc foo(...) = ...`) is a distinct top-level node, but the routine has a ## SINGLE `sdef`, emitted at the IMPLEMENTATION site (`sym.infoImpl`) — so the ## prototype's own position would otherwise vanish from the `.bif`. Tee it into ## the `deps` side-channel as a POSITIONED `(sig @proto )`: the loader ## skips the `sig` tag (processTopLevel), but `idetools.scanDef` finds the ## `SymbolDef` and reports the enclosing tag's line info — so a `--def` on a ## forward-declared proc returns TWO results (prototype + implementation), which ## is desired. Safe against symbol resolution: the loader rebuilds its name->pos ## table from the CONTENT body (`buildPosIndex`, written after `deps`, last write ## wins) so the real `sdef` still resolves; the extra on-disk index entry has no ## resolution consumer. The prototype's signature symbols (param names and the ## symbols in their type expressions) are teed too, positioned at the prototype, ## exactly as `emitSigOccurrences` records them for the implementation. # The `SymbolDef` carries the prototype line info too (not just the enclosing # tag): `scanDef` reads the position from the tag, but pass-1 `findPos` matches # a token by its OWN line info, so this is what makes a query issued AT the # prototype position resolve the symbol. let protoInfo = trLineInfo(w, n[namePos].info) let sid = pool.syms.getOrIncl(w.toNifSymName(sym)) w.deps.addParLe sigTag, protoInfo w.deps.addSymDef sid, protoInfo # scanDef reports this as a def w.deps.addSymUse sid, protoInfo # findPos (pass 1) / scanUses match a Symbol use w.deps.addParRi if sfFromGeneric notin sym.flagsImpl and paramsPos < n.safeLen: emitSigOccurrences(w, n[paramsPos]) proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) = if n == nil: dest.addDotToken else: if nfLazyBody in n.flags and forceLazyBodyHook != nil: # Materialize a deferred body before serializing so its real flags/typ and # children are written (never the empty `nfLazyBody` placeholder). forceLazyBodyHook(n) case n.kind of nkNone: assert n.typField == nil, "nkNone should not have a type" let info = trLineInfo(w, n.info) dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), info dest.addParRi of nkEmpty: if n.typField != nil: w.withNode dest, n: discard else: let info = trLineInfo(w, n.info) dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), info dest.addParRi of nkIdent: # nkIdent uses flags and typ when it is a generic parameter w.withNode dest, n: dest.addIdent n.ident.s of nkSym: writeSymNode(w, dest, n, n.sym) of nkCharLit: w.withNode dest, n: dest.add charToken(n.intVal.char, NoLineInfo) of nkIntLit .. nkInt64Lit: w.withNode dest, n: dest.addIntLit n.intVal of nkUIntLit .. nkUInt64Lit: w.withNode dest, n: dest.addUIntLit cast[BiggestUInt](n.intVal) of nkFloatLit .. nkFloat128Lit: w.withNode dest, n: dest.add floatToken(pool.floats.getOrIncl(n.floatVal), NoLineInfo) of nkStrLit .. nkTripleStrLit: w.withNode dest, n: dest.addStrLit n.strVal of nkNilLit: w.withNode dest, n: discard of nkLetSection, nkVarSection, nkConstSection: # Track local variables declared in let/var sections w.withNode dest, n: for child in n: addLocalSyms w, child # Process the child node writeNode(w, dest, child, forAst) of nkForStmt: # Track for loop variable (first child is the loop variable) w.withNode dest, n: if n.len > 0: addLocalSyms(w, n[0]) for i in 0 ..< n.len: writeNode(w, dest, n[i], forAst) of nkFormalParams: # Track parameters (first child is return type, rest are parameters) inc w.inProc w.withNode dest, n: for i in 0 ..< n.len: if i > 0: # Skip return type addLocalSyms(w, n[i]) writeNode(w, dest, n[i], forAst) dec w.inProc of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef, nkTemplateDef: # For top-level named routines (not forAst), just write the symbol. # The full AST will be stored in the symbol's sdef. if not forAst and n[namePos].kind == nkSym: let s = n[namePos].sym writeSym(w, dest, s) # A forward declaration is a SECOND top-level node for `s` (body-less here; # the real body — and the lone sdef — lands at the implementation). Tee the # prototype's own position so goto-def / find-usages surface it as well. let impl = s.astImpl if n.safeLen > bodyPos and n[bodyPos].kind == nkEmpty and impl != nil and impl != n and impl.safeLen > bodyPos and impl[bodyPos].kind != nkEmpty: emitFwdDecl(w, n, s) else: # Writing AST inside sdef or anonymous proc: write full structure inc w.inProc var ast = n var skipParams = false if n[namePos].kind == nkSym: ast = n[namePos].sym.astImpl if ast == nil: ast = n else: # params can only be recovered from `sym.typ.n` if the routine # was actually semchecked. A routine nested in a TEMPLATE body # (e.g. faststreams' `proc consumer(bytesVar: openArray[byte]) # {.gensym.}` inside `consumeOutputs`) has a sym but a nil type — # its params exist only in the AST; dropping them broke the # template-param substitution at expansion ("undeclared # identifier" for the injected name). skipParams = n[namePos].sym.typImpl != nil w.withNode dest, ast: for i in 0 ..< ast.len: if i == paramsPos and skipParams: # The dropped params still hold the source positions and the WRITTEN # type names (before alias/type resolution); tee them into the `deps` # side-channel for goto-def / find-usages (see `emitSigOccurrences`). # Skip generic INSTANCES: their param syms are instance-specific, and # the generic's own signature already records the source occurrences. if sfFromGeneric notin n[namePos].sym.flagsImpl: emitSigOccurrences(w, ast[i]) # Parameters are redundant with s.typ.n (and re-emitting their syms # is dangerous for generic instances — we do not adapt the symbols # properly). Emit an `nkEmpty` placeholder rather than a dot token: # a dot loads back as a `nil` son, but ast children must be real # nodes — the loaded routine ast is walked by passes (lambdalifting, # liftdestructors, transf) that dereference `ast[paramsPos]`, and # `nkEmpty` is the canonical empty slot. The actual params are # recovered from `sym.typ.n` where needed. dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)), NoLineInfo dest.addParRi else: writeNode(w, dest, ast[i], forAst) dec w.inProc of nkLambda, nkDo: # Lambdas are expressions, always write full structure inc w.inProc var ast = n if n[namePos].kind == nkSym: ast = n[namePos].sym.astImpl if ast == nil: ast = n w.withNode dest, ast: for i in 0 ..< ast.len: writeNode(w, dest, ast[i], forAst) dec w.inProc of nkImportStmt: if w.inProc > 0: # An `import` inside a template/macro/proc body — e.g. stew/importops' # `tryImport`: `when compiles((; import v)): import v`. It is part of the # body AST and must be serialized as a real node so the template # re-expands it at each use site; it is NOT a module-level dependency # edge (the import resolves where the template expands, against that # module's deps). Diverting it to `w.deps` (the top-level path below) # dropped it entirely: its child is the unexpanded template parameter # `v`, not a module sym, so `trImport` wrote nothing and the body # round-tripped EMPTY — a NIF-loaded `tryImport` then imported nothing. w.withNode dest, n: for i in 0 ..< n.len: writeNode(w, dest, n[i], forAst) else: # top-level import: recorded as a dependency edge — `importer.nim` has # already transformed `n` to contain a list of module syms. trImport w, n of nkIncludeStmt: trInclude w, n of nkExportStmt, nkExportExceptStmt: # Note: nkExportExceptStmt is transformed to nkExportStmt by semExportExcept, # but we handle both just in case trExport w, n else: w.withNode dest, n: for i in 0 ..< n.len: writeNode(w, dest, n[i], forAst) proc writeGlobal(w: var Writer; dest: var IcBuilder; n: PNode) = case n.kind of nkVarTuple: writeNode(w, dest, n) of nkIdentDefs, nkConstDef: # nkIdentDefs: [ident1, ident2, ..., type, default] # All children except the last two are identifiers for i in 0 ..< max(0, n.len - 2): writeGlobal(w, dest, n[i]) of nkPostfix: writeGlobal(w, dest, n[1]) of nkPragmaExpr: writeGlobal(w, dest, n[0]) of nkSym: writeSym(w, dest, n.sym) else: discard proc writeGlobals(w: var Writer; dest: var IcBuilder; n: PNode) = w.withNode dest, n: for child in n: writeGlobal(w, dest, child) proc writeToplevelNode(w: var Writer; dest, bottom: var IcBuilder; n: PNode) = case n.kind of nkStmtList, nkStmtListExpr: for son in n: writeToplevelNode(w, dest, bottom, son) of nkEmpty: discard "ignore" of nkTypeSection, nkCommentStmt, nkMixinStmt, nkBindStmt, nkUsingStmt, nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef, nkTemplateDef: # We write purely declarative nodes at the bottom of the file writeNode(w, bottom, n) of nkPragma: # Top-level pragmas — chiefly `{.emit.}`, plus the `{.push/pop.}` that guard # its neighbours — must survive the backend reload so the `cg` stage re-runs # genPragma/genEmit. The bottom (implementation) section is reloaded lazily # BY SYMBOL INDEX, which a symbol-less pragma can never be on, so a pragma # written there is silently dropped on reload (e.g. a module-level `#include` # vanishes and the generated C fails to compile). The header init section is # replayed verbatim by `processTopLevel`, so write it there instead. writeNode(w, dest, n) of nkConstSection: writeGlobals(w, bottom, n) of nkLetSection, nkVarSection: writeGlobals(w, dest, n) else: writeNode w, dest, n proc createStmtList(buf: var IcBuilder; info: PackedLineInfo) {.inline.} = buf.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), info buf.addDotToken # flags buf.addDotToken # type proc writeOp(w: var Writer; content: var IcBuilder; op: LogEntry) = case op.kind of HookEntry: case op.op of attachedDestructor: content.addParLe repDestroyTag, NoLineInfo of attachedAsgn: content.addParLe repCopyTag, NoLineInfo of attachedWasMoved: content.addParLe repWasMovedTag, NoLineInfo of attachedDup: content.addParLe repDupTag, NoLineInfo of attachedSink: content.addParLe repSinkTag, NoLineInfo of attachedTrace: content.addParLe repTraceTag, NoLineInfo of attachedDeepCopy: content.addParLe repDeepCopyTag, NoLineInfo content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() of ConverterEntry: content.addParLe repConverterTag, NoLineInfo content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() of MethodEntry: content.addParLe repMethodTag, NoLineInfo content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() of EnumToStrEntry: content.addParLe repEnumToStrTag, NoLineInfo content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() of PureEnumEntry: content.addParLe repPureEnumTag, NoLineInfo content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() of GenericInstEntry: discard "will only be written later to ensure it is materialized" # --------------------------- Interface cookie --------------------------- # # Port of Nimony's `processForChecksum` (dist/nimony/src/lib/nifindexes.nim): # ONE checksum per module over the importer-visible surface, stored in a tiny # `.iface.nif` sidecar written OnlyIfChanged. deps.nim points the # dependents' `nim_m` build edges at the sidecar instead of the bulky semmed # NIF, so nifmake's mtime pruning stops the m-step cascade at the first # module whose interface did not change. # # Hashed (importer-visible surface): # - import/include/export entries, `(replay ...)` macro-cache actions and the # rep* hook/converter/enumtostr registrations (all eagerly consumed by every # importer's sem via processTopLevel/loadTransitiveHooks). # - every EXPORTED `(sd ...)`: full content for consts/types/vars/lets; for # EVERY routine kind (plain procs, templates, macros, iterators, generics, # `inline` procs alike) only the SIGNATURE — the body is skipped. A routine # body is invisible to a dependent's SEM unless the dependent expands / # instantiates / VM-runs it, and each of those records a NeedsImpl (strong) # edge gating the dependent on this module's IMPL cookie instead (see # `cookieSd`). This keeps the iface cookie body-insensitive, so a body edit # re-sems only the modules that actually consumed that body — not every # importer (the old model folded inline-semantics bodies into the iface # cookie, re-semming all importers on any such body edit). # - nothing else: private defs and top-level init code are invisible to # importers' sem (their effects on dependents' CODEGEN — and the codegen # effect of inline iterator/proc body edits — are covered by the nifc # backend's transitive NIF-mtime invalidation, which is unchanged). # # Token-content hashing only — line infos never enter the hash. Names DEFINED # inside a hashed (sd) (params, locals, the embedded `(td `tK.item.mod)` defs) # are replaced by per-sd ordinals and module-local `tK.item references are # replaced by their structural td hash: both carry process-local mint counters # that shift file-wide when an unrelated body creates a new type (measured: # a single new instantiation renumbered every later signature), while # dependents never reference them by name (verified over a full compiler # cache: cross-module refs hit only top-level routine names). # # The cookie finally mixes in the DIRECT dependencies' sidecar contents # ("hash chaining"): an interface change then propagates transitively # level-by-level even when an intermediate module's own surface is unchanged # (its sem still consumed the dep's surface, e.g. via the transitive hook # replay). Chaining also guarantees a fired rule refreshes its sidecar mtime, # which nifmake's max-output `needsRebuild` needs to not re-fire forever. # # The IMPL cookie (`.impl.nif`) complements it: a line-info-free hash # of the module's ENTIRE content with the iface cookie mixed in. Dependents # that consumed this module's bodies at compile time (recorded in the # `.edges.nif` sidecar; see `ModuleGraph.icImplDeps`) are gated on it instead. type CookieCtx = object selfSuffix: string tdRanges: Table[uint32, int] # td sym -> start of its first (td ...) tree memo: Table[uint32, string] # td sym -> structural digest expanding: HashSet[uint32] # cycle guard for recursive td expansion depSuffixes: seq[string] # module suffixes of the direct imports proc nextTree(flat: seq[CookieTok]; i: int): int = ## Index just past the atom or balanced subtree starting at `i`. result = i+1 if flat[i].kind != ckParLe: return var nested = 0 var j = i while j < flat.len: case flat[j].kind of ckParLe: inc nested of ckParRi: dec nested if nested == 0: return j+1 else: discard inc j result = flat.len proc updateAtom(s: var Sha1State; t: CookieTok) = # mirrors nimony's nifchecksums.update: token content only, no line infos case t.kind of ckParLe: s.update "(" s.update t.tag of ckParRi: s.update ")" of ckIdent: s.update " " s.update t.str of ckStr: s.update " \"" s.update t.str of ckInt: s.update " " s.update $t.ival of ckUInt: s.update " " s.update $t.uval of ckFloat: # hash the bit pattern, not a formatted float (no formatting variance) s.update " f" s.update $cast[uint64](t.fval) of ckChar: s.update " c" s.update $t.cval of ckDot: s.update "." of ckSym, ckSymDef: discard "handled by hashRegion" proc isModuleLocalName(c: CookieCtx; name: string): bool = let sn = parseSymName(name) result = sn.module.len == 0 or sn.module == c.selfSuffix proc hashRegion(s: var Sha1State; c: var CookieCtx; flat: seq[CookieTok]; start, theEnd: int; skipFrom = -1; skipTo = -1; keepFirstDefLiteral = false) proc expandTd(c: var CookieCtx; flat: seq[CookieTok]; name: uint32; nameStr: string): string = ## Structural digest of a module-local type def: hashes the `(td ...)` tree ## instead of the volatile `tK.item counter name. Memoized; cycles fall back ## to the literal name (sound — at worst a spurious cookie change). if c.memo.hasKey(name): return c.memo[name] if not c.tdRanges.hasKey(name) or c.expanding.contains(name): return nameStr c.expanding.incl name let start = c.tdRanges[name] var sub = newSha1State() hashRegion(sub, c, flat, start, nextTree(flat, start)) result = "&" & $SecureHash(sub.finalize()) c.expanding.excl name c.memo[name] = result proc hashRegion(s: var Sha1State; c: var CookieCtx; flat: seq[CookieTok]; start, theEnd: int; skipFrom = -1; skipTo = -1; keepFirstDefLiteral = false) = # pass 1: assign ordinals to every symbol DEFINED in the hashed region # (params, locals, embedded type defs). The region's own top-level name # (first SymbolDef) stays literal when requested — it is what importers # reference. var ords = initTable[uint32, int]() var first = keepFirstDefLiteral var i = start while i < theEnd: if i == skipFrom: i = skipTo continue if flat[i].kind == ckSymDef: let sym = flat[i].sym if first: first = false elif isModuleLocalName(c, flat[i].name) and not ords.hasKey(sym): ords[sym] = ords.len inc i # pass 2: hash first = keepFirstDefLiteral i = start while i < theEnd: if i == skipFrom: i = skipTo continue let t = flat[i] if t.kind in {ckSym, ckSymDef}: let sym = t.sym let name = t.name s.update(if t.kind == ckSymDef: " :" else: " ") if t.kind == ckSymDef and first: first = false s.update name elif ords.hasKey(sym): s.update "%" s.update $ords[sym] elif name.startsWith("`t") and isModuleLocalName(c, name): s.update expandTd(c, flat, sym, name) else: s.update name else: updateAtom s, t inc i proc cookieSd(s: var Sha1State; c: var CookieCtx; flat: seq[CookieTok]; start: int): int = ## Contributes one `(sd ...)` subtree to the cookie; returns the index past it. result = nextTree(flat, start) if flat[start+1].kind != ckSymDef: return let marker = flat[start+2] if not (marker.kind == ckIdent and marker.str == "x"): return # not importable -> invisible to dependents' sem (nimony parity) # field layout, see writeSymDef: kind magic flags options offset position # annex type owner ast loc constraint instantiatedFrom var fields: array[13, int] = default(array[13, int]) var i = start + 3 for f in 0 ..< 13: fields[f] = i i = nextTree(flat, i) var kind = skUnknown {.cast(uncheckedAssign).}: kind = parse(TSymKind, flat[fields[0]].tag) var skipFrom = -1 var skipTo = -1 if kind in routineKinds: # Routines contribute their SIGNATURE only to the iface cookie. A routine # body is invisible to a dependent's SEM unless the dependent expands, # instantiates, or VM-runs it — and each of those records a NeedsImpl # (strong) edge that gates the dependent on this module's IMPL cookie # instead (templates -> semTemplateExpr, generics -> generateInstance, # macros/compile-time procs -> the VM's genProc, getImpl -> opcGetImpl). # Inline iterators and `inline`-callconv procs are inlined at codegen; the # nifc backend's transitive NIF-mtime invalidation re-codegens their users. # So no routine body needs to live in the iface cookie. let ast = fields[9] if flat[ast].kind == ckParLe: # skip son `bodyPos` (6) of the routine ast tree; NOT the last element — # sem appends the result sym at `resultPos` (7) after the body. let astEnd = nextTree(flat, ast) var p = ast + 1 # the flags atom var ok = true for _ in 0 ..< 2 + bodyPos: # flags, type, sons 0..5 p = nextTree(flat, p) if p >= astEnd - 1: ok = false break if ok: skipFrom = p skipTo = nextTree(flat, p) # non-routine kinds (consts carry their value, types their structure incl. # default field values): hash everything. hashRegion(s, c, flat, start, result, skipFrom, skipTo, keepFirstDefLiteral = true) proc scanStmtsForCookie(s: var Sha1State; c: var CookieCtx; flat: seq[CookieTok]) = ## Walks the whole written module, hashing only the importer-visible pieces; ## unknown structure is descended into (var/let/type section wrappers, ## top-level code) but contributes nothing itself — nimony-style. let exportName = toNifTag(nkExportStmt) let exportExceptName = toNifTag(nkExportExceptStmt) var i = 0 while i < flat.len: let t = flat[i] if t.kind == ckParLe: let tg = t.tag if tg == symDefTagName: i = cookieSd(s, c, flat, i) elif tg == "implementation": i = nextTree(flat, i) elif tg == "replay" or tg == "repconverter" or tg == "repdestroy" or tg == "repwasmoved" or tg == "repcopy" or tg == "repsink" or tg == "repdup" or tg == "reptrace" or tg == "repdeepcopy" or tg == "repenumtostr" or tg == "repmethod" or tg == exportName or tg == exportExceptName or tg == "include": let e = nextTree(flat, i) hashRegion(s, c, flat, i, e) i = e elif tg == "import": let e = nextTree(flat, i) hashRegion(s, c, flat, i, e) for j in i ..< e: if flat[j].kind == ckStr: let suffix = flat[j].str if suffix notin c.depSuffixes: c.depSuffixes.add suffix i = e else: inc i # descend without hashing else: inc i proc icGroupSuffixes(config: ConfigRef): HashSet[string] = ## Module suffixes of the --icGroup cycle members compiled by this very ## process (their sidecars are being produced concurrently, so neither ## chaining nor edge recording may depend on them). result = initHashSet[string]() for p in config.icGroup: result.incl cachedModuleSuffix(config, fileInfoIdx(config, AbsoluteFile p)) proc writeCookieFile(config: ConfigRef; selfSuffix, tag, hex, ext: string) = # Binary NIF cookie `(tag "hex")`, content-stable so an unchanged hash keeps the # sidecar mtime (nifmake prunes the re-sem cascade behind it). var b = newIcBuilder(4) b.openTag tag b.addStrLit hex b.closeTag() let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ext).string storeBifStable(b, path, "." & extractModuleSuffix(path)) proc writeIfaceCookie(config: ConfigRef; thisModule: int32; flat: seq[CookieTok]): string = let selfSuffix = modname(thisModule, config) var c = CookieCtx(selfSuffix: selfSuffix) # pre-pass: first (td ...) occurrence per type name, wherever it is embedded var i = 0 while i < flat.len: if flat[i].kind == ckParLe and flat[i].tag == typeDefTagName and i+1 < flat.len and flat[i+1].kind == ckSymDef: let nm = flat[i+1].sym if not c.tdRanges.hasKey(nm): c.tdRanges[nm] = i inc i var s = newSha1State() scanStmtsForCookie(s, c, flat) # chain the direct deps' cookies; co-members of an --icGroup cycle are # excluded (their sidecars are being produced by this very rule — chaining # them would make the hash depend on within-group write order). let groupSuffixes = icGroupSuffixes(config) for dep in c.depSuffixes: if dep == selfSuffix or dep in groupSuffixes: continue let depIface = toGeneratedFile(config, AbsoluteFile(dep), ".iface.bif").string s.update "|" s.update dep s.update ":" s.update(try: readFile(depIface) except IOError, OSError: "") result = $SecureHash(s.finalize()) writeCookieFile(config, selfSuffix, "iface", result, ".iface.bif") proc writeImplCookie(config: ConfigRef; thisModule: int32; flat: seq[CookieTok]; ifaceHex: string) = ## The implementation cookie: a line-info-free hash of the module's ENTIRE ## serialized content (private defs and routine bodies included), with the ## module's own iface cookie mixed in so impl sensitivity is a strict ## superset of iface sensitivity (incl. the chained dep ifaces — a NeedsImpl ## edge REPLACES the iface edge, it must not lose its triggers). Dependents ## that consumed this module's bodies at compile time are gated on this file ## instead of the iface cookie. Comment-only edits move neither cookie. ## No id normalization here: a counter shift implies some real content ## change elsewhere in the module, which flips the hash anyway — and any ## body change is exactly what NeedsImpl dependents must see. let selfSuffix = modname(thisModule, config) var s = newSha1State() for t in flat: if t.kind in {ckSym, ckSymDef}: s.update(if t.kind == ckSymDef: " :" else: " ") s.update t.name else: updateAtom s, t s.update "|iface:" s.update ifaceHex writeCookieFile(config, selfSuffix, "impl", $SecureHash(s.finalize()), ".impl.bif") proc writeEdgesFile(config: ConfigRef; thisModule: int32; implDeps: seq[int]) = ## Records which modules' bodies this compilation consumed at compile time ## (`ModuleGraph.icImplDeps`): the NeedsImpl edge set. deps.nim reads this ## sidecar when regenerating the build file and gates this module on those ## dependencies' IMPL cookies instead of their iface cookies. let selfSuffix = modname(thisModule, config) let groupSuffixes = icGroupSuffixes(config) var suffixes: seq[string] = @[] for id in implDeps: if id == thisModule.int: continue let suffix = cachedModuleSuffix(config, FileIndex id) if suffix.len == 0 or suffix == selfSuffix or suffix in groupSuffixes: continue if suffix notin suffixes: suffixes.add suffix sort suffixes # Native nifcore writer (Stage 2): `(edges "suffix" ...)`, byte-identical. var b = newIcBuilder(4 + 2*suffixes.len) b.openTag "edges" for suffix in suffixes: b.addStrLit suffix b.closeTag() let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".edges.bif").string # Deliberately ALWAYS written (unlike every other output of the nim_m rule): # nothing gates on this file's mtime — deps.nim only reads its content — so # it doubles as the rule's freshness stamp. nifmake's `needsRebuild` takes # the freshest output as proof of "ran since the inputs changed"; without an # always-written output a rule whose re-run produces only content-identical # (mtime-preserved) files would re-fire on every warm build (e.g. after an # edit was reverted). Nimony's analog is its always-written `.s.bif`. storeBif(b, path, "." & extractModuleSuffix(path)) proc writeSemDeps*(config: ConfigRef; thisModule: int32; importPaths: seq[string]) = ## The module's REAL direct imports as `nim m` sem resolved them — static ## plus any a macro generated — recorded as full source paths. `nim ic` reads ## this `.s.deps.nif` to re-derive the build graph: imports the static scanner ## missed become new nodes (replacing the old build-failure discovery loop), ## and `when false` imports the scanner over-included are pruned. Always ## written so it is current after every successful sem (like `.edges`). ## ## Ported to nifcore: delegates to `icnifcore.writeSemDeps` (Stage 1 of the ## NIF-stack migration; see doc/ic_nifcore_port.md). Output is byte-identical ## to the previous `nifstreams` writer. icnifcore.writeSemDeps(config, thisModule, importPaths) proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; opsLog: seq[LogEntry]; replayActions: seq[PNode] = @[]; implDeps: seq[int] = @[]; reexportedModules: seq[(string, string)] = @[]; genericOffers: seq[tuple[generic, inst: PSym; concreteTypes: seq[PType]; genericParamsCount: int]] = @[]; typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[]; resolvedImportDeps: seq[FileIndex] = @[]; firstUnusedId: int32 = 0; expansions: seq[(PSym, TLineInfo)] = @[]) = var w = Writer(infos: newLineInfoWriter(config), currentModule: thisModule) w.deps = newIcBuilder(64) var content = newIcBuilder(300) let rootInfo = trLineInfo(w, n.info) createStmtList(content, rootInfo) # Write replay actions first, wrapped in a (replay ...) node if replayActions.len > 0: content.addParLe replayTag, rootInfo for action in replayActions: writeNode(w, content, action) content.addParRi() # Only write ops that belong to this module for op in opsLog: if op.module == thisModule.int: writeOp(w, content, op) var bottom = newIcBuilder(300) w.writeToplevelNode content, bottom, n # Resolved import edges that left no syntactic `import` node in the top-level # AST: an import generated INSIDE a `when` condition (e.g. stew/importops' # `when tryImport x:` -> `when compiles((; import x)): import x`) really # imports `x` — `addImportFileDep` recorded the edge in `graph.importDeps` — # but the import node is folded away with the condition, so `trImport` never # saw it and the NIF `deps` section omitted it. The backend closure walk # (nifbackend.loadBackendModules) follows NIF `deps`, so without this edge a # template-imported module's `{.compile.}`/`{.passL.}` directives never replay # and its C/asm objects go unlinked (undefined `hashtree_hash`/`my_c_add` at # link). Emit any resolved edge not already written as a syntactic import. for f in resolvedImportDeps: let fp = moduleSuffix(config, f) if not w.depSuffixes.containsOrIncl(fp): w.deps.addParLe importTag, NoLineInfo w.deps.addDotToken # flags w.deps.addDotToken # type w.deps.addStrLit fp w.deps.addParRi # Re-exported MODULES (`import x; export x`): semExport puts only x's # member syms into the nkExportStmt; the module sym itself reaches the # exporter's interface via `reexportSym` and acts as a QUALIFIER there # (`asmm.x86.nd`). Serialize (name, suffix) pairs so the loader can # rebuild that part of the interface. for (mname, msuffix) in reexportedModules: w.deps.addParLe reexpModTag, NoLineInfo w.deps.addStrLit mname w.deps.addStrLit msuffix w.deps.addParRi # Generic-instance OFFERS: every generic instance this module created # (`getOrDefault[MultiCodec]`, …). A consumer that re-instantiates the same # generic must REUSE this instance instead of re-running `instantiateBody` in # its own module scope — which lacks symbols visible only at the generic's # definition site (e.g. a distinct type's `==` from the type's module), so # operator/mixin resolution would fail ("type mismatch" at `hashcommon.rawGet`). # The loader (modulegraphs.moduleFromNifFile) rebuilds `procInstCache` from # these so `genericCacheGet` hits and the wrong-scope re-instantiation is # skipped. Layout: (offer ...). for off in genericOffers: w.deps.addParLe offerTag, NoLineInfo w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.generic)), NoLineInfo w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.inst)), NoLineInfo w.deps.addIntLit off.genericParamsCount for ct in off.concreteTypes: w.deps.addSymUse pool.syms.getOrIncl(typeToNifSym(ct, w.infos.config)), NoLineInfo w.deps.addParRi # Record this module's own absolute source path. The NIF suffix is a hash of # the (relative) path (gear2/modnames.moduleSuffix) and is NOT reversible, so # the standalone include-graph scanner (`scanIncludeGraph`, used by nimsuggest # cold queries) needs the path written explicitly to map an included file back # to the *source* of its includer without loading the module. w.deps.addParLe modulesrcTag, NoLineInfo w.deps.addStrLit toFullPath(config, FileIndex(thisModule)) w.deps.addParRi # Template/macro expansions leave no trace in the sem'checked AST, so record # each as `(expansion )`: a `Symbol` use of the expanded # routine carrying the ORIGINAL call-site line info. The loader skips the tag # (processTopLevel), but `idetools` scans every `Symbol` token in the buffer, # so this restores "find usages / goto-def" for templates and macros. for (sym, info) in expansions: if sym == nil: continue w.deps.addParLe expansionTag, NoLineInfo w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), trLineInfo(w, info) w.deps.addParRi # Generic TYPE-instance OFFERS: the `tyGenericInst` types this module created # (e.g. `HashArray[8192, Gwei]`). Non-IC keeps ONE such instance in the global # `typeInstCache`, so a structural bound computed at the first instantiation # site (e.g. an `array[…]` bound that depends on a `mixin`/`compiles()` whose # resolution differs by import scope) is frozen and reused everywhere. A # separate `nim m` process never repopulates `typeInstCache` from NIFs, so it # re-instantiates in its own scope and can compute a DIFFERENT bound (the SSZ # `dataPerChunk` divergence). The loader rebuilds `g.typeInstCache` from these # so `semtypinst.searchInstTypes` hits and reuses the baked instance. # Layout: (toffer ). for off in typeOffers: # Carry the generic body sym and the instance type as STRING LITERALS, not # SymUse tokens: `addSymUse` rewrites a same-module reference into the NIF # "local form" (suffix stripped, resolved by the content loader against the # module being read), but this offer lives in the `deps` header and is read # by a CONSUMER with no such module context. The full names round-trip # verbatim as strings and `createTypeStub`/`resolveHookSym` resolve them # directly (cf. `loadImport`, which carries module suffixes the same way). w.deps.addParLe typeOfferTag, NoLineInfo w.deps.addStrLit w.toNifSymName(off.generic) w.deps.addStrLit typeToNifSym(off.inst, w.infos.config) w.deps.addParRi # OWNER MUST EMIT: a type reachable only through an offered instance — the # `concreteTypes` of an offered proc instance (e.g. chronicles `writeValue[T]`, # where `T` is this module's own object type) or an offered generic type # instance — may never be reached by the normal top-level serialization above. # If this module OWNS such a type, force-emit its typedef so that a consumer # which reuses the offer can resolve the cross-module SymUse to it. Without this # the consumer writes `t..` and the loader asserts # `symbol has no offset`. `writeType` emits the def (and recurses into owned # sons) only for an own, still-Complete type; an already-Sealed one is skipped. for off in genericOffers: for ct in off.concreteTypes: if ct != nil and ct.uniqueId.module == w.currentModule and ct.state == Complete: writeType(w, bottom, ct) for off in typeOffers: if off.inst != nil and off.inst.uniqueId.module == w.currentModule and off.inst.state == Complete: writeType(w, bottom, off.inst) # the implTag is used to tell the loader that the # bottom of the file is the implementation of the module: content.addParLe implTag, NoLineInfo content.addParRi() addAll(content, bottom) content.addParRi() let m = modname(w.currentModule, w.infos.config) let bifPath = completeGeneratedFilePath(config, AbsoluteFile(m).changeFileExt(".s.bif")).string var dest = newIcBuilder(600) createStmtList(dest, rootInfo) # First child: the backend id seed (see `(unusedid)` / readUnusedId). dest.addParLe unusedIdTag, NoLineInfo dest.addIntLit firstUnusedId.int64 dest.addParRi() addAll(dest, w.deps) # do not write the (stmts .. ) wrapper: addStmtsBody(dest, content) # ensure the hooks we announced end up in the NIF file regardless of # whether they have been used: for op in opsLog: if op.module == thisModule.int: let s = op.sym if s.state != Sealed: s.state = Sealed if restoresWrittenState(config): w.writtenSyms.add s writeSymDef w, dest, s dest.addParRi() # Nimsuggest and normal code generation reuse these symbols/types as live, # mutable targets. Sealing is only needed for intra-emit dedup; once the NIF # is built, un-seal them. The guard stays in force for a real `nim m` build. if restoresWrittenState(config): for s in w.writtenSyms: if s.state == Sealed: s.state = Complete for t in w.writtenTypes: if t.state == Sealed: t.state = Complete # OnlyIfChanged keeps the mtime of content-identical rewrites: nifmake's # mtime-based `needsRebuild` then prunes the rebuild cascade level by # level, and the nifc backend can trust "semmed NIF older than the cnif # artifact" as an honest per-module unchanged stamp. # CONTENT-STABLE (`storeBifStable`, the bif analogue of the old text `.s.nif`'s # `OnlyIfChanged`): when `nim m` re-runs (e.g. it was scheduled because a sibling # input churned) but produces byte-identical sem output, the `.s.bif` mtime MUST # be preserved, else every dependent backend stage sees its input as "newer" and # rebuilds — with whatever compiler this run uses. In a self-rebuild (`bootic`) # that re-translates only SOME modules with the new compiler while others reuse # the prior compiler's artifacts → a MIXED binary that needs a 3rd fixed-point # iteration to wash out. The `nim m` rule still has its always-written run-marker: # the `.edges.bif` (writeEdgesFile), so a no-op re-run does not re-fire. # Step 3: emit the compact binary NIF as the SOLE on-disk module artifact (no # text `.s.nif` twin — writing two files per module only slows the build; debug # a `.bif` via `tools/bif2nif`). Re-homed into a private fresh pool # so the file holds only THIS module's literals. storeBifStable(dest, bifPath, "." & extractModuleSuffix(bifPath)) if not isDefined(config, "icNoIfaceGate"): var flat = flattenForCookie(dest) let ifaceHex = writeIfaceCookie(config, thisModule, flat) writeImplCookie(config, thisModule, flat, ifaceHex) writeEdgesFile(config, thisModule, implDeps) # --------------------------- Loader (lazy!) ----------------------------------------------- # Step 2b reader shims over nifcore cursors: template info(n: Cursor): NifLineInfo = rawLineInfo(n) ## line info of the token at `n` (was the inline `n.info` of nifcursors). template cursorTag(n: Cursor): string = n.tags.tagName(cursorTagId(n)) ## tag name of the TagLit at `n` — for VALUE uses only (parse into an enum, ## error messages). For tag *checks* use `tagIs`. Resolved via the cursor's OWN ## tag pool (`n.tags`), not the shared `icTags`: a `bif`-loaded module carries a ## fresh per-file tag pool whose ids only line up with its own `tagName`. template tagIs(n: Cursor; name: string): bool = n.tags.tagName(cursorTagId(n)) == name ## True iff `n`'s TagLit is the IC tag `name`. A string compare against the ## cursor's own tag pool — id comparison against a process-global cache is ## impossible once `bif` mints fresh per-file tag pools (ids are per-pool). proc nodeKind(n: Cursor): TNodeKind {.inline.} = assert n.kind == TagLit parse(TNodeKind, cursorTag(n)) proc expect(n: Cursor; k: set[nifcore.NifKind]) = if n.kind notin k: when defined(debug): writeStackTrace() quit "[NIF decoder] expected: " & $k & " but got: " & $n.kind proc expect(n: Cursor; k: nifcore.NifKind) {.inline.} = expect n, {k} proc firstSon*(n: Cursor): Cursor {.inline.} = ## Non-consuming peek at the first child of a TagLit. The `inc` is on a copy, ## so it never advances the caller's cursor. result = n inc result proc loadBool(n: var Cursor): bool = if n.kind == TagLit: result = tagIs(n, "true") n.into: discard else: raiseAssert "(true)/(false) expected" type NifModule = ref object buf: TokenBuf # the WHOLE module, parsed eagerly (Step 2: replaces the # lazy byte-offset stream entirely — symbol/type loading # AND the body reader now cursor over this resident buffer) symCounter: int32 # seeded from the file's `(unusedid)` so backend syms # start above every frontend/lowered id (no collision) typeCounter: int32 # ditto for backend TYPES (closure envs etc.) index: Table[string, NifIndexEntry] # name -> entry; `offset` is a TOKEN # position in `buf` (was a byte offset) suffix: string loweredPrimary: bool # `buf` is the lowered `.t.bif` (cg/emit stage). The lower # stage never changes type DEFINITIONS, so they are NOT # carried in `.t.bif`; a type def not in `index` is read # from the `.s.bif` companion below (loaded on demand). semBuf: TokenBuf # the `.s.bif` (semchecked) buffer — TYPE-def fallback semIndex: Table[string, NifIndexEntry] semTried: bool # `semBuf`/`semIndex` load attempted (idempotent) PendingBody = object ## A deferred routine body (bodyPos son). `cursor` points AT the body node in ## the module buffer (kept alive by the cursor's refcounted owner); `localSyms` ## is the snapshot of the enclosing sym def's local symbols so body-local ## references resolve to the SAME PSyms the signature already created. cursor: Cursor thisModule: string localSyms: Table[string, PSym] DecodeContext* = object infos: LineInfoWriter pendingBodies: Table[int, PendingBody] # nodeId(placeholder) -> deferred body #moduleIds: Table[string, int32] types: Table[string, (PType, NifIndexEntry)] syms: Table[string, (PSym, NifIndexEntry)] mods: Table[FileIndex, NifModule] cache: IdentCache mainModuleSuffix: string ## Mangled module name of the module being compiled fresh (cmdM). Symbols ## belonging to it that are re-exported by a dependency must NOT be loaded ## as stubs, otherwise they collide with the freshly compiled originals. symLoads, typeLoads: CountTable[FileIndex] ## Diagnostics (opt-in via env `NIM_IC_LOADSTATS`): per OWNING-module count ## of stub materializations in THIS process. Quantifies the "every backend ## worker deserializes system.bif + a bunch of others" cost — breadth (how ## many syms) attributed to duplication axis (which shared module). proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable result = DecodeContext(infos: newLineInfoWriter(config), cache: cache) var loadStatsInit {.threadvar.}: int # 0=unknown 1=on 2=off var statsCtxPtr {.threadvar.}: ptr DecodeContext var loaderCtx {.threadvar.}: ptr DecodeContext # the live `program`; for lazy-body # materialization off the len hook var nodesDecoded {.threadvar.}: int # all PNodes materialized this proc var astFieldNodes {.threadvar.}: int # subset: routine-body (s.ast) subtrees proc dumpLoadStatsExit() {.noconv.} = if statsCtxPtr == nil: return let c = statsCtxPtr var merged = initTable[FileIndex, array[2, int]]() for m, cnt in c.symLoads.pairs: merged.mgetOrPut(m, [0, 0])[0] = cnt for m, cnt in c.typeLoads.pairs: merged.mgetOrPut(m, [0, 0])[1] = cnt var order: seq[FileIndex] = @[] var totS, totT: int = 0 for m, a in merged: order.add m totS += a[0]; totT += a[1] sort(order, proc (a, b: FileIndex): int = (merged[b][0] + merged[b][1]) - (merged[a][0] + merged[a][1])) let params = commandLineParams() let target = if params.len > 0: params[^1] else: "?" stderr.writeLine "=== IC loadstats pid=" & $getCurrentProcessId() & " main=" & c.mainModuleSuffix & " target=" & target & " ===" stderr.writeLine " TOTAL symLoads=" & $totS & " typeLoads=" & $totT & " modulesTouched=" & $order.len let pct = if nodesDecoded > 0: 100 * astFieldNodes div nodesDecoded else: 0 stderr.writeLine " PNODES decoded=" & $nodesDecoded & " routineBody=" & $astFieldNodes & " (" & $pct & "% deferrable via lazy PSym.ast)" for m in order: let a = merged[m] let name = if c.mods.hasKey(m): c.mods[m].suffix else: "?" stderr.writeLine " " & $(a[0] + a[1]) & "\tsym=" & $a[0] & " typ=" & $a[1] & "\t" & name proc recordLoad(c: var DecodeContext; m: FileIndex; isType: bool) = if loadStatsInit == 0: loadStatsInit = if existsEnv("NIM_IC_LOADSTATS"): 1 else: 2 if loadStatsInit == 1: statsCtxPtr = addr c addExitProc(dumpLoadStatsExit) if loadStatsInit == 2: return if isType: c.typeLoads.inc(m) else: c.symLoads.inc(m) proc nextBackendSymItem*(c: var DecodeContext; module: int32): int32 = ## Allocate the next backend-minted SYM item for `module` from the SAME ## per-module counter the loader uses when it re-homes `@bk` syms loaded from ## the module's `.t.bif` (loadSymStub/extractLocalSymsFromTree). The `lower` ## stage serializes its lifted hooks/temps as `@bk` syms, and cg mints MORE ## backend syms (RTTI destroy wrappers, ...) into the same module. Both are ## keyed by `.id` (= `toId(itemId)`) in `declaredThings`/`declaredProtos`, so ## if the two id producers (the loader's `symCounter` and cg's idgen) ran ## independently they could mint the same item: e.g. a `rttiDestroy` wrapper ## and the very `=destroy` hook it wraps both land on backend item 21 -> one ## masks the other in `declaredThings` -> the hook's body is never emitted -> ## "undefined reference" at link. Drawing every backend sym from this one ## counter keeps them disjoint. Returns -1 if the module is not loaded yet ## (then the caller falls back to the idgen's own counter — only reachable ## for sem-time `@bk` minting, whose module is never loaded in that process). let fi = module.FileIndex if not c.mods.hasKey(fi): return -1'i32 let p = addr c.mods[fi].symCounter inc p[] result = p[] proc nextBackendTypeItem*(c: var DecodeContext; module: int32): int32 = ## TYPE analogue of `nextBackendSymItem`: the `lower`/`cg` stages mint fresh ## backend TYPES (closure-env objects, ptr wrappers) whose itemId must not ## collide with the module's loaded types. Drawn from the per-module ## `typeCounter`, which `moduleId` seeds from the file's `(unusedid)` so the ## first minted type sits ABOVE every frontend/lowered type item. Returns -1 ## if the module is not loaded (caller falls back to the idgen's own counter). let fi = module.FileIndex if not c.mods.hasKey(fi): return -1'i32 let p = addr c.mods[fi].typeCounter inc p[] result = p[] proc setMainModule*(c: var DecodeContext; fileIdx: FileIndex) = ## Records the module that is being compiled fresh so that re-exports of its ## own symbols by dependencies are not turned into duplicate stubs. c.mainModuleSuffix = modname(fileIdx.int, c.infos.config) proc getMainModuleSuffix*(c: DecodeContext): string {.inline.} = c.mainModuleSuffix proc loadedState(c: DecodeContext): ItemState {.inline.} = ## State to give a freshly loaded symbol or type. During the C code generation ## phase (`nim nifc`) the backend (lambda lifting, the transformer, etc.) ## legitimately mutates the loaded entities and never writes them back to a NIF, ## so they must be mutable (`Complete`). nimsuggest (`ideActive`) is the same ## case: it reuses loaded symbols as live query targets and mutates them during ## sem and suggestion bookkeeping (usage tracking, flags) without authoritatively ## writing those mutations back (its NIF emits are gated to non-dirty, error-free ## modules and re-serialize from the proper state). During a plain `nim m` ## semantic check a loaded entity belongs to an already-compiled dependency and ## must stay `Sealed` so accidental mutations are caught. if c.infos.config.cmd == cmdNifC or c.infos.config.ideActive: Complete else: Sealed proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry): Cursor = ## Step 2a: O(1) cursor into the module's resident `buf` at the def's token ## position. No I/O, no per-symbol materialization — and, because each call ## returns an INDEPENDENT cursor, none of the old stream-cursor clobber hazards ## (the `jumpTo(saved)` save/restore dance) apply anymore. result = cursorAt(c.mods[module].buf, entry.offset) type LoadFlag* = enum LoadFullAst, AlwaysLoadInterface proc isGlobalIndexSym(s, dottedSuffix: string): bool = ## Mirror of `nifbuilder.addSymbolDefRetIsGlobal` / `bif.isGlobalSymbol`: a sym ## gets an index entry when its name — with a self-module `dottedSuffix` ## compressed to one trailing dot — has >= 2 dots (counting from index 1). var lim = s.len if dottedSuffix.len > 0 and s.endsWith(dottedSuffix): lim = s.len - dottedSuffix.len + 1 if lim > s.len: lim = s.len var dots = 0 for i in 1 ..< lim: if s[i] == '.': inc dots dots >= 2 proc buildPosIndex(buf: var TokenBuf; suffix: string): Table[string, NifIndexEntry] = ## Step 2a token-position index: scan the eagerly-parsed module `buf` for the ## global `SymbolDef`s it OWNS and record each at the token position of its ## enclosing tag (`(sd`/`(td`), with visibility from the marker that follows ## the def. Replaces `readEmbeddedIndex` (whose byte offsets are meaningless ## once the file is parsed); mirrors `bif.buildIndex` and the text writer's ## `(.index …)`. Foreign symbols appear only as `Symbol` uses (never ## `SymbolDef`s) so they are naturally excluded. result = initTable[string, NifIndexEntry]() let dotted = "." & suffix if buf.len == 0: return var c = buf.beginRead() var mostRecentTagPos = 0 while c.hasMore: case c.kind of TagLit: mostRecentTagPos = cursorToPosition(buf, c) inc c # descend into the body (visit every token) of SymbolDef: let nm = symName(c) let tagPos = mostRecentTagPos inc c # advance to the marker / next sibling if isGlobalIndexSym(nm, dotted): let vis = if c.hasMore and c.kind == DotToken: Hidden else: Exported result[nm] = NifIndexEntry(offset: tagPos, info: NoLineInfo, vis: vis) else: inc c proc readUnusedId(buf: var TokenBuf): int32 = ## Find the module's `(unusedid )` directive — emitted as the FIRST child ## of the top-level `(stmts ...)` by writeNifModule/writeLoweredModule — and ## return its value (the first free itemId). 0 if absent (older artifact: the ## backend then falls back to its own un-seeded counter, i.e. pre-`unusedid` ## behaviour). result = 0'i32 if buf.len == 0: return var c = buf.beginRead() if c.kind != TagLit: return # outermost (stmts ...) inc c # descend into stmts body while c.hasMore: if c.kind == TagLit: if tagName(c.tags, c.cursorTagId) == "unusedid": inc c # into the unusedid body if c.hasMore and c.kind == IntLit: result = int32 intVal(c) return else: skip c # not it; skip this whole subtree else: inc c proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): FileIndex = var isKnownFile = false result = c.infos.config.registerNifSuffix(suffix, isKnownFile) # Always load the module's index if it's not already in c.mods # This is needed when resolving symbols from modules that were registered elsewhere # but haven't had their NIF index loaded yet let hasEntry = c.mods.hasKey(result) if not hasEntry or AlwaysLoadInterface in flags: # Module artifacts are binary NIF (`.bif`). The `cg`/`emit` backend stages # load the LOWERED whole-module `.t.bif` (transformed bodies + lambda-lifted # signatures/entities baked in by the `lower` stage — see writeLoweredModule); # the `lower` stage and the frontend (`cmdM`) load the semchecked `.s.bif`. # This mirrors `toNifFilename` (kept in sync). `bif.load` mints FRESH per-file # pools, so the buffer's literals/tags resolve through its own # `cursorPool(n)`/`n.tags` (the reader is pool-agnostic); the token-position # index is rebuilt name-based via `buildPosIndex`. let conf = c.infos.config let useLowered = conf.cmd == cmdNifC and (conf.icBackendStage == "cg" or conf.icBackendStage == "emit") var modFile = (getNimcacheDir(conf) / RelativeFile(suffix & ".t.bif")).string let lowered = useLowered and fileExists(modFile) if not lowered: modFile = (getNimcacheDir(conf) / RelativeFile(suffix & ".s.bif")).string if not fileExists(modFile): raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile & ". This can happen when loading a module from NIF that references another module " & "whose NIF file hasn't been written yet." var m = bif.load(modFile) let index = buildPosIndex(m.buf, suffix) # Seed the backend id counters ABOVE every id the file already uses, so a # freshly-minted backend sym/type (closure env, RTTI hook, temp) can never # share a `toId` with a loaded one. See `readUnusedId` / `(unusedid)`. let seed = readUnusedId(m.buf) c.mods[result] = NifModule(buf: ensureMove m.buf, index: index, suffix: suffix, symCounter: seed, typeCounter: seed, loweredPrimary: lowered) proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry = let ii = addr c.mods[module].index result = ii[].getOrDefault(nifName) if result.offset == 0: raiseAssert "symbol has no offset: " & nifName proc ensureSemBuf(c: var DecodeContext; module: FileIndex) = ## Lazily load the module's `.s.bif` companion (`semBuf`/`semIndex`) for the TYPE ## fallback. Only meaningful when the primary `buf` is the lowered `.t.bif`, which ## omits frontend type defs (the lower stage never changes them). Idempotent. let m = c.mods[module] if m.semTried: return m.semTried = true let semFile = (getNimcacheDir(c.infos.config) / RelativeFile(m.suffix & ".s.bif")).string if not fileExists(semFile): return var sm = bif.load(semFile) m.semIndex = buildPosIndex(sm.buf, m.suffix) m.semBuf = ensureMove sm.buf proc hasTypeOffset(c: var DecodeContext; module: FileIndex; nifName: string): bool = ## Does a TYPE def for `nifName` exist for `module` — in the primary buffer, or ## (lowered primary) the `.s.bif` companion? result = false let m = c.mods[module] if m.index.getOrDefault(nifName).offset != 0: return true if m.loweredPrimary: ensureSemBuf(c, module) result = m.semIndex.getOrDefault(nifName).offset != 0 proc typeCursor(c: var DecodeContext; module: FileIndex; nifName: string): Cursor = ## A cursor at a TYPE's `(td …)` def: the primary buffer if present (an `@bk` ## closure-env type minted by the lower stage, or any `.s.bif`-primary module), ## else the `.s.bif` companion (frontend type defs are NOT carried in `.t.bif`). let m = c.mods[module] let e = m.index.getOrDefault(nifName) if e.offset != 0: return cursorAt(m.buf, e.offset) if m.loweredPrimary: ensureSemBuf(c, module) let se = m.semIndex.getOrDefault(nifName) if se.offset != 0: return cursorAt(m.semBuf, se.offset) raiseAssert "symbol has no offset: " & nifName proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PNode proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]) proc reconstructSysType(c: var DecodeContext; name: string; k: int; itemVal: int32): PType = ## Rebuild a module-less magic singleton (see `SysModuleSuffix`) from its kind ## alone — it has no fields and no `.nif` to load. Cached in `c.types` so all ## references in this decode context share one instance. result = c.types.getOrDefault(name)[0] if result == nil: let id = itemId(-1'i32, itemVal) result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Complete) if TTypeKind(k) == tyNil: result.sizeImpl = c.infos.config.target.ptrSize result.alignImpl = int16 c.infos.config.target.ptrSize c.types[name] = (result, NifIndexEntry()) proc stripBkSuffix(rawMod: string): (bool, string) {.inline.} = ## Split a possibly-`@bk` (BackendLocalMarker) module suffix into ## `(isBackendMinted, realSuffix)`. See `toNifSymName`/`nifTypeName`. if rawMod.endsWith(BackendLocalMarker): (true, rawMod[0 ..< rawMod.len - BackendLocalMarker.len]) else: (false, rawMod) proc nextSymId(c: var DecodeContext; module: FileIndex; isBk: bool): ItemId = ## Mint the next per-module SYM id from `symCounter`: a `backendItemId` for a ## process-local `@bk` sym, else a plain loader `itemId`. Both draw from the one ## counter so loaded and cg-minted backend syms stay disjoint (see ## `nextBackendSymItem`). Types do NOT use this — they preserve the item parsed ## from their own name (see `tryCreateTypeStub`). let val = addr c.mods[module].symCounter inc val[] result = if isBk: backendItemId(module.int32, val[]) else: itemId(module.int32, val[]) proc mintSymId(c: var DecodeContext; rawMod: string): (FileIndex, ItemId) = ## Resolve a possibly-`@bk` module suffix to its FileIndex and mint a fresh sym ## id for it — the common case where the module is not needed before minting ## (see `stripBkSuffix`/`nextSymId`). let (isBk, realMod) = stripBkSuffix(rawMod) let module = moduleId(c, realMod) result = (module, c.nextSymId(module, isBk)) proc makePartialSymStub(c: var DecodeContext; symAsStr: string; sn: ParsedSymName; id: ItemId; entry: NifIndexEntry): PSym = ## Create + cache (keyed by the NIF name) a `Partial` global-sym stub, lazily ## filled later by `loadSym` from `entry`. `stubKindAndName` strips NIF-only ## markers (e.g. a package's `PkgMarker`) so the backend mangles the clean name. let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, entry) proc tryCreateTypeStub(c: var DecodeContext; name: string): PType = ## Like `createTypeStub` but returns nil instead of raising when the type has ## no offset in its module index (used by the best-effort `(offer …)` loader). ## Step 2b: takes the sym NAME string (pool-agnostic) — the reader never juggles ## a nifcore/nifstreams `SymId`. if not name.startsWith("`t"): return nil result = c.types.getOrDefault(name)[0] if result == nil: var i = len("`t") var k = 0 while i < name.len and name[i] in {'0'..'9'}: k = k * 10 + name[i].ord - ord('0') inc i if i < name.len and name[i] == '.': inc i var itemVal = 0'i32 while i < name.len and name[i] in {'0'..'9'}: itemVal = itemVal * 10'i32 + int32(name[i].ord - ord('0')) inc i if i < name.len and name[i] == '.': inc i let suffix = name.substr(i) if suffix == SysModuleSuffix: return reconstructSysType(c, name, k, itemVal) let (isBk, realSuffix) = stripBkSuffix(suffix) let modIdx = moduleId(c, realSuffix).int32 let id = if isBk: backendItemId(modIdx, itemVal) else: itemId(modIdx, itemVal) let modFi = id.module.FileIndex if not hasTypeOffset(c, modFi, name): return nil result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) # `loadType` re-resolves the buffer via `typeCursor`, so the cached entry is a # don't-care for types — store the primary one if any (else a 0-offset stub). c.types[name] = (result, c.mods[modFi].index.getOrDefault(name)) proc createTypeStub(c: var DecodeContext; name: string): PType = ## As `tryCreateTypeStub`, but a missing index offset is a hard error (the ## caller demanded a definition that must exist). assert name.startsWith("`t") result = tryCreateTypeStub(c, name) if result == nil: raiseAssert "symbol has no offset: " & name proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]) = ## Scan a tree for local symbol definitions (sdef tags) and add them to localSyms. ## For local symbols, fully load them immediately since they have no index offsets. ## After this proc returns, n is positioned AFTER the tree. # Atoms (non-compound nodes): nothing to scan, just skip past them. if n.kind != TagLit: skip n return if tagIs(n, typeDefTagName): # A nested inline type owns its own field name-scope: its fields are object-LOCAL # symbols (``f.`) that can collide name+position with a sibling/outer # type's field (e.g. astdef's `TLoc.flags` vs `TNode.flags`, both inline). Do NOT # pull them into this scope; `loadTypeFromCursor` loads each type's reclist in an # isolated `localSyms`. skip n return if tagIs(n, symDefTagName): # Found an sdef - check if it's a new local symbol. let name = n.firstSon expect name, SymbolDef let symName = symName(name) let sn = parseSymName(symName) if sn.module.len == 0 and symName notin localSyms: # Local symbol - create stub and immediately load it fully # since local symbols have no index offsets for lazy loading let module = moduleId(c, thisModule) let id = c.nextSymId(module, isBk = false) # `stubKindAndName` strips NIF-only markers (e.g. a field's `` `f ``) so the # backend mangles the clean name; `loadSymFromCursor` then fills the real kind. let (_, stubName) = stubKindAndName(c.cache, sn.name) let sym = PSym(itemId: id, kindImpl: skStub, name: stubName, disamb: sn.count.int32, state: Complete) localSyms[symName] = sym # `loadSymFromCursor` enters the `(sd` and consumes the whole block, # leaving n positioned after the closing `)`. loadSymFromCursor(c, sym, n, thisModule, localSyms) sym.state = c.loadedState # mark as fully loaded return # Otherwise descend into every child, scanning each for nested local sdefs. n.loopInto: extractLocalSymsFromTree(c, n, thisModule, localSyms) proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms: var Table[string, PSym]) proc loadTypeStub(c: var DecodeContext; n: var Cursor; localSyms: var Table[string, PSym]): PType = if n.kind == DotToken: result = nil skip n elif n.kind == Symbol: result = createTypeStub(c, symName(n)) skip n elif n.kind == TagLit and tagIs(n, typeDefTagName): result = createTypeStub(c, symName(n.firstSon)) if result.state == Partial: result.state = c.loadedState # Mark as loaded to prevent loadType from re-loading with empty localSyms # A type's reclist is its own field name-scope: object-local field names # (``f.`) can collide name+position with the enclosing scope's or a # sibling inline type's fields. Load it in an isolated `localSyms`. var typeLocalSyms = initTable[string, PSym]() loadTypeFromCursor(c, n, result, typeLocalSyms) else: skip n # Type already loaded, skip over the td block else: raiseAssert "type expected but got " & $n.kind proc loadFieldStub(c: var DecodeContext; symAsStr: string; thisModule: string; localSyms: var Table[string, PSym]; typ: PType = nil): PSym = ## A cross-context object-field reference (see `FieldMarker`): its def lives in ## the owning type's reclist (a different seek, absent from this body's ## `localSyms`), and it has no module suffix / index entry. There is nothing to ## resolve — `cgen.genRecordField` re-navigates the object type's reclist by ## `name` (`lookupFieldAgain`/`lookupInRecord`), so the use-site field need only ## carry the clean field name (+ position for tuples, + type so a lower-stage ## transform that builds a fresh node off this sym still re-serializes a type). ## NOT shared across uses: each carries its own `typ`, and two distinct fields can ## share a local name+position (cross-type), so a shared stub would mistype one. let sn = parseSymName(symAsStr) let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) let module = moduleId(c, thisModule) # `sn.count` is the field POSITION (see toNifSymName): tuple element access reads # it directly off this stub, so preserve it. Named-object uses re-navigate by name. result = PSym(itemId: c.nextSymId(module, isBk = false), kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Complete) result.positionImpl = sn.count.int32 if typ != nil: result.typImpl = typ proc loadSymStub(c: var DecodeContext; symAsStr: string; thisModule: string; localSyms: var Table[string, PSym]): PSym = let sn = parseSymName(symAsStr) # For local symbols (no module suffix), they MUST be in localSyms. # Local symbols are not in the index - they're defined inline in the NIF file. # If not found, it's a bug in how we populate localSyms. if sn.module.len == 0: result = localSyms.getOrDefault(symAsStr) if result != nil: return result elif sn.name.endsWith(FieldMarker): # A cross-context object-field reference reaching a non-dotExpr slot (e.g. a # `{.guard.}` field, an owner): stub it like any other field use. return c.loadFieldStub(symAsStr, thisModule, localSyms) else: raiseAssert "local symbol '" & symAsStr & "' not found in localSyms." # Global symbol - look up in index for lazy loading result = c.syms.getOrDefault(symAsStr)[0] if result == nil: # A process-local backend sym (closure env field / `:env` param) is named # `…@bk`: `mintSymId` homes it to that module with a # backendItemId so it stays disjoint from the loader's real id space. let (module, id) = c.mintSymId(sn.module) let offs = c.mods[module].index.getOrDefault(symAsStr) if offs.offset == 0: # Only module/package self-syms are never written as `(sd)` entries, so a # missing index offset means this is such a sym — typically the OWNER of an # `include`d symbol (`.0.`). Synthesize a resolvable # skModule stub (itemId item-0 = the module self-sym) instead of asserting # "symbol has no offset". `Complete` so accessors never try to lazy-load it. result = PSym(itemId: itemId(module.int32, 0'i32), kindImpl: skModule, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, infoImpl: newLineInfo(module, 1, 1), state: Complete) c.syms[symAsStr] = (result, NifIndexEntry()) return result result = c.makePartialSymStub(symAsStr, sn, id, offs) proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PSym = if n.kind == DotToken: result = nil skip n elif n.kind == Symbol: result = loadSymStub(c, symName(n), thisModule, localSyms) skip n elif n.kind == TagLit and tagIs(n, symDefTagName): let s = symName(n.firstSon) skip n result = loadSymStub(c, s, thisModule, localSyms) else: raiseAssert "sym expected but got " & $n.kind & ( if n.kind == Ident: " '" & strVal(n) & "'" else: "") proc isStub*(t: PType): bool {.inline.} = t.state == Partial proc isStub*(s: PSym): bool {.inline.} = s.state == Partial proc loadAtom[T](t: typedesc[set[T]]; n: var Cursor): set[T] = if n.kind == DotToken: result = {} skip n else: expect n, Ident result = parse(T, strVal(n)) skip n proc loadAtom[T: enum](t: typedesc[T]; n: var Cursor): T = if n.kind == DotToken: result = default(T) skip n else: expect n, Ident result = parse(T, strVal(n)) skip n proc loadAtom(t: typedesc[string]; n: var Cursor): string = expect n, StrLit result = strVal(n) skip n proc loadAtom[T: int16|int32|int64](t: typedesc[T]; n: var Cursor): T = expect n, IntLit result = intVal(n).T skip n template loadField(field) {.dirty.} = field = loadAtom(typeof(field), n) proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = loadField loc.k loadField loc.storage loadField loc.flags loadField loc.snippet proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms: var Table[string, PSym]) = expect n, TagLit if not tagIs(n, typeDefTagName): raiseAssert "(td) expected" var scanCursor = n # copy cursor at start of type var typesModule = parseSymName(symName(n.firstSon)).module if typesModule.endsWith(BackendLocalMarker): # A backend-minted (`@bk`) type's name carries the marker in its module part; # strip it so the nested-local pre-scan resolves the real module, not a # nonexistent `@bk.nif`. typesModule = typesModule[0 ..< typesModule.len - BackendLocalMarker.len] extractLocalSymsFromTree(c, scanCursor, typesModule, localSyms) n.into: # enter (td, body consumes all children, closing ) is consumed by `into` expect n, SymbolDef # ignore the type's name, we have already used it to create this PType's itemId! skip n expect n, DotToken skip n #loadField t.kind loadField t.flagsImpl loadField t.callConvImpl loadField t.sizeImpl loadField t.alignImpl loadField t.paddingAtEndImpl t.itemId = itemId(t.itemId.module, loadAtom(int32, n)) # nonUniqueId if n.kind == StrLit: # itemId.module differs from uniqueId.module (an `exactReplica` of a # foreign type): restore the canonical module half t.itemId = itemId(int32(moduleId(c, strVal(n))), t.itemId.item) skip n elif n.kind == DotToken: skip n t.typeInstImpl = loadTypeStub(c, n, localSyms) t.nImpl = loadNode(c, n, typesModule, localSyms) t.ownerFieldImpl = loadSymStub(c, n, typesModule, localSyms) t.symImpl = loadSymStub(c, n, typesModule, localSyms) loadLoc c, n, t.locImpl while n.hasMore: t.sonsImpl.add loadTypeStub(c, n, localSyms) proc loadType*(c: var DecodeContext; t: PType) = if t.state != Partial: return t.state = c.loadedState recordLoad(c, t.itemId.module.FileIndex, isType = true) # A backend-minted (`@bk`) closure-env type produced by the `lower` stage lives # ONLY in the `.t.nif` and is keyed by its `@bk` name (see nifTypeName), not the # canonical `typeToNifSym` (which asserts non-`@bk`). Reconstruct that name so a # Partial `@bk` stub that escaped the inline pre-scan can still be force-loaded. let typeName = if t.uniqueId.isBackendMinted: "`t" & $ord(t.kind) & "." & $t.uniqueId.item & "." & modname(t.itemId.module, c.infos.config) & BackendLocalMarker else: typeToNifSym(t, c.infos.config) let modFi = t.itemId.module.FileIndex # `typeCursor` resolves to the primary `.t.bif` (`@bk` env types) or falls back to # the `.s.bif` companion (frontend type defs, which `.t.bif` no longer carries). var n = typeCursor(c, modFi, typeName) var localSyms = initTable[string, PSym]() loadTypeFromCursor(c, n, t, localSyms) proc loadAnnex(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PLib = if n.kind == DotToken: result = nil skip n elif n.kind == TagLit: result = PLib(kind: parse(TLibKind, cursorTag(n))) n.into: result.generated = loadBool(n) result.isOverridden = loadBool(n) expect n, StrLit result.name = strVal(n) skip n result.path = loadNode(c, n, thisModule, localSyms) else: raiseAssert "`lib/annex` information expected" proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]) = ## Loads a symbol definition. The cursor must be positioned AT the opening ## `(sd` TagLit; `into` consumes the whole sdef including its closing `)`. n.into: expect n, SymbolDef # ignore the symbol's name, we have already used it to create this PSym instance! skip n if n.kind == Ident: if strVal(n) == "x": s.flagsImpl.incl sfExported skip n else: raiseAssert "expected `x` as the export marker" elif n.kind == DotToken: skip n else: raiseAssert "expected `x` or '.' but got " & $n.kind expect n, TagLit {.cast(uncheckedAssign).}: s.kindImpl = parse(TSymKind, cursorTag(n)) if s.kindImpl == skPackage and s.name.s.endsWith(PkgMarker): # Fallback: stubs are normally created with the clean name already # (see stubKindAndName); strip the NIF-only marker if one slipped through. s.name = c.cache.getIdent(s.name.s[0 ..< s.name.s.len - PkgMarker.len]) n.into: # the (kind ...) sub-block case s.kindImpl of skLet, skVar, skField, skForVar: s.guardImpl = loadSymStub(c, n, thisModule, localSyms) loadField s.bitsizeImpl loadField s.alignmentImpl else: discard loadField s.magicImpl loadField s.flagsImpl loadField s.optionsImpl loadField s.offsetImpl if s.kindImpl == skModule: expect n, DotToken skip n var isKnownFile = false s.positionImpl = int c.infos.config.registerNifSuffix(thisModule, isKnownFile) # do to the precompiled mechanism things end up as main modules which are not! excl s.flagsImpl, sfMainModule else: loadField s.positionImpl s.annexImpl = loadAnnex(c, n, thisModule, localSyms) # Local symbols were already extracted upfront in loadSym, so we can use # the simple loadTypeStub here. s.typImpl = loadTypeStub(c, n, localSyms) s.ownerFieldImpl = loadSymStub(c, n, thisModule, localSyms) # Load the AST for routine symbols and constants # Constants need their AST for astdef() to return the constant's value let astNodesBefore = nodesDecoded s.astImpl = loadNode(c, n, thisModule, localSyms) if loadStatsInit == 1 and s.kindImpl in routineKinds: astFieldNodes += nodesDecoded - astNodesBefore loadLoc c, n, s.locImpl s.constraintImpl = loadNode(c, n, thisModule, localSyms) s.instantiatedFromImpl = loadSymStub(c, n, thisModule, localSyms) # The TRANSFORMED body slot (see writeSymDef). It means DIFFERENT things by # which file `moduleId` loaded (see toNifFilename): # * `cg`/`emit` read `.t.bif` — the slot is the `lower` stage's AUTHORITATIVE # lowered body; ALWAYS load it so `transformBody` short-circuits and the # backend NEVER re-derives (the whole point of the artifact). # * the `lower` stage reads `.s.bif` — the slot is the VM/CT lowering sem # cached; load it only when REUSE is on (`icReuseSemLowering`), else leave # `transformedBody` nil so the `lower` stage re-derives from the pristine # body (the 2026-06-27 simplicity spec, doc/ic_backend_simplify.md §6). # * frontend `cmdM` never loads it (a dependent needs no foreign lowered body, # and reconstructing one must not perturb effect/exception inference). let conf = c.infos.config let loadSlot = s.kindImpl in routineKinds and conf.cmd == cmdNifC and (conf.icBackendStage == "cg" or conf.icBackendStage == "emit" or (conf.icBackendStage == "lower" and icReuseSemLowering(conf))) if loadSlot: s.transformedBodyImpl = loadNode(c, n, thisModule, localSyms) else: skip n proc loadSym*(c: var DecodeContext; s: PSym) = if s.state != Partial: return s.state = c.loadedState if loaderCtx == nil: loaderCtx = addr c recordLoad(c, s.itemId.module.FileIndex, isType = false) let symsModule = s.itemId.module.FileIndex let nifname = globalName(s, c.infos.config) var n = cursorFromIndexEntry(c, symsModule, c.syms[nifname][1]) expect n, TagLit if not tagIs(n, symDefTagName): raiseAssert "(sd) expected" # Pre-scan the ENTIRE symbol definition to extract ALL local symbols upfront. # This ensures local symbols are registered before any references to them, # regardless of where they appear in the definition (in types, nested procs, etc.) var localSyms = initTable[string, PSym]() var scanCursor = n extractLocalSymsFromTree(c, scanCursor, c.mods[symsModule].suffix, localSyms) # Now parse the symbol definition with all local symbols pre-registered s.infoImpl = c.infos.oldLineInfo(n.info, cursorPool(n)) # The `##` doc comment (if any) rides as a NIF comment on the sym def token; # capture it before advancing, then restore it onto the loaded AST so that # suggest's `extractDocComment` (findDocComment on `s.ast`) finds it. let docId = rawLineInfo(n).comment # nifcore StrId of the `#..#` doc comment let docPool = cursorPool(n) # the buffer's own strings pool (shared or bif-fresh) loadSymFromCursor(c, s, n, c.mods[symsModule].suffix, localSyms) if uint32(docId) != 0'u32 and s.astImpl != nil and nodeCommentWriter != nil: nodeCommentWriter(s.astImpl, docPool.strings[docId]) proc sealLoadedRoutines*(c: var DecodeContext) = ## Before `writeLoweredModule` re-serializes the lowered module, seal ONLY the ## module's ROUTINE syms. A `.t.nif` written by `writeLoweredModule` is the ## SOLE source the `cg` stage loads (there is no `.s.nif` fallback for its ## bodies), so every type, global, param and local must still emit a REAL def ## in it — only cross-routine references may be `SymUse`s (each routine's def is ## emitted once, at module scope, by the explicit stub loop). Types/globals stay ## `Complete` so `writeType`/`writeGlobals` emit them; routines become `Sealed` ## so a body referencing another routine writes a `SymUse` resolved through the ## module index. for _, v in c.syms: if v[0] != nil and v[0].state == Complete and v[0].kindImpl in routineKinds: v[0].state = Sealed proc resolveHookSym*(c: var DecodeContext; name: string): PSym template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info, cursorPool(n)) result = newNodeI(kind, info) n.into: result.flags = loadAtom(TNodeFlags, n) result.typField = c.loadTypeStub(n, localSyms) body proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PNode = if loadStatsInit == 1: inc nodesDecoded result = nil case n.kind of Symbol: let info = c.infos.oldLineInfo(n.info, cursorPool(n)) let symName = symName(n) # Check local symbols first let localSym = localSyms.getOrDefault(symName) if localSym != nil: result = newSymNode(localSym, info) skip n elif isFieldNifName(symName): # Cross-context object-field reference: stub a `skField` from the local name # (see `loadFieldStub`). The field's type is recovered from the object type at # codegen time, so this leaf carries no type of its own. result = newSymNode(c.loadFieldStub(symName, thisModule, localSyms), info) result.flags.incl nfLazyType skip n else: result = newSymNode(c.loadSymStub(n, thisModule, localSyms), info) if result.typField == nil: result.flags.incl nfLazyType of DotToken: result = nil skip n of StrLit: result = newStrNode(strVal(n), c.infos.oldLineInfo(n.info, cursorPool(n))) skip n of TagLit: let kind = n.nodeKind case kind of nkNone: # special NIF introduced tag? if tagIs(n, hiddenTypeTagName): n.into: let typ = c.loadTypeStub(n, localSyms) let info = c.infos.oldLineInfo(n.info, cursorPool(n)) var s: PSym if n.kind == Symbol and isFieldNifName(symName(n)): # Field SymUse wrapped with its explicit type (see writeSymNode): stub # the field, carrying the wrapper's type on BOTH the node and the sym so # a lower-stage transform that builds a fresh node off the sym still has a # type to re-serialize. s = c.loadFieldStub(symName(n), thisModule, localSyms, typ) skip n else: s = c.loadSymStub(n, thisModule, localSyms) result = newSymNode(s, info) result.typField = typ elif tagIs(n, symDefTagName): let info = c.infos.oldLineInfo(n.info, cursorPool(n)) let name = n.firstSon assert name.kind == SymbolDef let symName = symName(name) # Check if this is a local symbol (no module suffix in name) let sn = parseSymName(symName) let isLocal = sn.module.len == 0 var sym: PSym # In every branch below `n` stays at the `(sd` TagLit; `loadSymFromCursor` # enters and consumes the whole block, and `skip n` consumes it wholesale. if isLocal: # Local symbol - not in the index, defined inline in NIF. # Check if we already have a stub from extractLocalSymsFromType sym = localSyms.getOrDefault(symName) if sym == nil: # First time seeing this local symbol - create it let module = moduleId(c, thisModule) let id = c.nextSymId(module, isBk = false) # strip NIF-only markers (a field's `` `f ``) so the backend sees the # clean name; `loadSymFromCursor` below fills the real kind. let (_, stubName) = stubKindAndName(c.cache, sn.name) sym = PSym(itemId: id, kindImpl: skStub, name: stubName, disamb: sn.count.int32, state: Complete) localSyms[symName] = sym # register for later references # Now fully load the symbol from the sdef loadSymFromCursor(c, sym, n, thisModule, localSyms) sym.state = c.loadedState # mark as fully loaded result = newSymNode(sym, info) elif sn.module.endsWith(BackendLocalMarker): # A backend-minted (`@bk`) def lives ONLY inline in this `.t.nif` body # (not in any module index): create/find its cached stub and FILL it # from the sdef instead of skipping (which would leave the skModule/ # Partial stub `loadSymStub` made unresolved). sym = c.loadSymStub(symName, thisModule, localSyms) if sym.state == Partial: sym.state = c.loadedState loadSymFromCursor(c, sym, n, thisModule, localSyms) else: skip n result = newSymNode(sym, info) result.flags.incl nfLazyType else: # A module-homed inline sdef. Normally its def lives in that module's # index and is loaded lazily, so we skip the inline copy. BUT a # transform-created closure-env FIELD (`x0.0.clo`) is module-homed yet # lives ONLY inline in this `.t.nif` reclist — it has no index entry. # Skipping it leaves a nil-typed `skModule` fallback stub (from # loadSymStub's "no offset" path) and codegen of the env struct then # dereferences a nil field type. Detect the unindexed case and FILL # the sym from the inline def instead. let m = moduleId(c, sn.module) let indexed = c.mods[m].index.hasKey(symName) if indexed: sym = c.loadSymStub(symName, thisModule, localSyms) skip n # skip the entire sdef for indexed symbols else: sym = c.syms.getOrDefault(symName)[0] if sym == nil: sym = PSym(itemId: c.nextSymId(m, isBk = false), kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[symName] = (sym, NifIndexEntry()) sym.state = c.loadedState loadSymFromCursor(c, sym, n, thisModule, localSyms) result = newSymNode(sym, info) result.flags.incl nfLazyType elif tagIs(n, typeDefTagName): raiseAssert "`td` tag in invalid context" elif tagIs(n, "none"): result = newNodeI(nkNone, c.infos.oldLineInfo(n.info, cursorPool(n))) n.into: result.flags = loadAtom(TNodeFlags, n) else: raiseAssert "Unknown NIF tag " & cursorTag(n) of nkEmpty: result = newNodeI(nkEmpty, c.infos.oldLineInfo(n.info, cursorPool(n))) n.into: if n.hasMore: result.flags = loadAtom(TNodeFlags, n) result.typField = c.loadTypeStub(n, localSyms) of nkIdent: let info = c.infos.oldLineInfo(n.info, cursorPool(n)) n.into: let flags = loadAtom(TNodeFlags, n) let typ = c.loadTypeStub(n, localSyms) expect n, Ident result = newIdentNode(c.cache.getIdent(strVal(n)), info) skip n result.flags = flags result.typField = typ of nkSym: #let info = c.infos.oldLineInfo(n.info, cursorPool(n)) #result = newSymNode(c.loadSymStub n, info) raiseAssert "nkSym should be mapped to a NIF symbol, not a tag" of nkCharLit: c.withNode n, result, kind: expect n, CharLit result.intVal = n.charLit.int skip n of nkIntLit .. nkInt64Lit: c.withNode n, result, kind: expect n, IntLit result.intVal = intVal(n) skip n of nkUIntLit .. nkUInt64Lit: c.withNode n, result, kind: expect n, UIntLit result.intVal = cast[BiggestInt](uintVal(n)) skip n of nkFloatLit .. nkFloat128Lit: c.withNode n, result, kind: if n.kind == FloatLit: result.floatVal = floatVal(n) skip n elif n.kind == TagLit: if tagIs(n, "inf"): result.floatVal = Inf elif tagIs(n, "nan"): result.floatVal = NaN elif tagIs(n, "neginf"): result.floatVal = NegInf else: raiseAssert "expected float literal but got " & cursorTag(n) n.into: discard else: raiseAssert "expected float literal but got " & $n.kind of nkStrLit .. nkTripleStrLit: c.withNode n, result, kind: expect n, StrLit result.strVal = strVal(n) skip n of nkNilLit: c.withNode n, result, kind: discard of routineDefs: # Defer the heavy `bodyPos` son: build the routine-def header eagerly, but # install a `nfLazyBody` placeholder (carrying the real body kind, so cheap # `ast[bodyPos].kind != nkEmpty` checks need no load) whose children are # materialized on demand (see `materializeLazyBody`, driven by the `len` # hook). An empty body is a single node — not worth deferring. c.withNode n, result, kind: var idx = 0 while n.hasMore: if idx == bodyPos and n.kind == TagLit and n.nodeKind notin {nkEmpty, nkNone}: let info = c.infos.oldLineInfo(n.info, cursorPool(n)) let ph = newNodeI(n.nodeKind, info) ph.flags.incl nfLazyBody c.pendingBodies[cast[int](ph)] = PendingBody(cursor: n, thisModule: thisModule, localSyms: localSyms) result.sons.add ph skip n else: result.sons.add c.loadNode(n, thisModule, localSyms) inc idx else: c.withNode n, result, kind: while n.hasMore: result.sons.add c.loadNode(n, thisModule, localSyms) else: raiseAssert "expected string literal but got " & $n.kind proc materializeLazyBody*(c: var DecodeContext; node: PNode) = ## Fill a `nfLazyBody` placeholder's children in place (identity-preserving: ## callers already hold `node`). Decodes the deferred body from the stashed ## cursor with the enclosing def's `localSyms` so param/local refs resolve to ## the SAME PSyms the signature created. node.flags.excl nfLazyBody # clear first: the loadNode below calls `len` let key = cast[int](node) var pb = PendingBody() if not c.pendingBodies.pop(key, pb): return var cur = pb.cursor let real = c.loadNode(cur, pb.thisModule, pb.localSyms) # `real` has the same kind as the placeholder (peeked at defer time); graft its # decoded content onto the node the callers hold. node.sons = real.sons node.typField = real.typField node.flags = real.flags forceLazyBodyHook = proc (n: PNode) {.nimcall, raises: [], tags: [], gcsafe.} = # `len` (the sole caller path) MUST stay effect-free, so this hook is typed # `raises: []`. The underlying `loadNode` chain infers `raises: [KeyError]` # (index/sym Table lookups), but materialization only ever runs for a body # DEFERRED during THIS load — the buffer/index is present by construction, so a # KeyError here means a corrupt cache: a fatal bug, not a recoverable error. # Treat it as effect-free (a `Defect`-like invariant) via a scoped cast. if loaderCtx != nil: {.cast(raises: []).}: {.cast(tags: []).}: {.cast(gcsafe).}: materializeLazyBody(loaderCtx[], n) proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; nifName: string; entry: NifIndexEntry; thisModule: string): PSym = ## Loads a symbol from the NIF index entry using the entry directly. ## Creates a symbol stub without looking up in the index (since the index may be moved out). result = c.syms.getOrDefault(nifName)[0] if result == nil: let sn = parseSymName(nifName) let rawMod = if sn.module.len > 0: sn.module else: thisModule let (_, id) = c.mintSymId(rawMod) result = c.makePartialSymStub(nifName, sn, id, entry) proc extractBasename(nifName: string): string = ## Extract the base name from a NIF name (ident.disamb.module -> ident) result = "" for c in nifName: if c == '.': break result.add c proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; interf, interfHidden: var TStrTable; thisModule: string) = ## Populates interface tables from the NIF index structure. ## Uses the simple embedded index for offsets, exports passed from processTopLevel. # Move the index table out to avoid iterator invalidation # (moduleId can add to c.mods which would invalidate Table iterators) var indexTab = move c.mods[module].index # Add all symbols to interf (exported interface) and interfHidden for nifName, entry in indexTab: if entry.vis == Exported: let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) if sym != nil: strTableAdd(interf, sym) strTableAdd(interfHidden, sym) elif not nifName.startsWith("`t"): # do not load types, they are not part of an interface but an implementation detail! #echo "LOADING SYM ", nifName, " ", entry.offset let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) if sym != nil: strTableAdd(interfHidden, sym) # Move index table back c.mods[module].index = move indexTab proc moduleSymbolStubs*(c: var DecodeContext; module: FileIndex): seq[PSym] = ## Stubs for every non-type symbol serialized in `module`'s NIF index. The ## per-module backend uses this to emit the routines a module OWNS: procs are ## serialized as `(sd ...)` symbol-defs and loaded lazily, never as ## `nkProcDef` statements in the top-level stmt list, so `genTopLevelStmt` ## alone never reaches them — without this, a routine called only from other ## modules would be emitted by nobody once the demanding module merely ## prototypes it. ## ## Returns lazy stubs: the index table is moved out while iterating (loading a ## symbol can register new modules and invalidate the iterator), so the caller ## forces full load (`.kind`, `.ast`) and filters AFTER this returns, with the ## index back in place. result = @[] if not c.mods.hasKey(module): return var indexTab = move c.mods[module].index let thisModule = c.mods[module].suffix for nifName, entry in indexTab: if nifName.startsWith("`t"): continue # types are not routines let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) if sym != nil: result.add sym c.mods[module].index = move indexTab proc loadedModuleTypes*(c: var DecodeContext; module: FileIndex): seq[PType] = ## Stubs for every TYPE this module owns — but, unlike before, WITHOUT force- ## loading them. `writeLoweredModule` emits a real def into the `.t.bif` only for ## the ones already `Complete` (= the lower stage actually loaded, hence possibly ## MUTATED — lambda-lifting flips a proc type to `ccClosure` and grows env types ## with captured fields). Every untouched type stays `Partial`, so a reference to ## it serializes as a `SymUse` that a cg/emit consumer resolves from the `.s.bif` ## (loader fallback `typeCursor`/`ensureSemBuf`) — the lower stage leaves those ## defs unchanged, so re-emitting them into `.t.bif` was pure cost. Collect names ## first: `createTypeStub` may register modules / mutate `c.types`, which must not ## invalidate the index iterator. result = @[] if not c.mods.hasKey(module): return var names: seq[string] = @[] for nifName in c.mods[module].index.keys: if nifName.startsWith("`t"): names.add nifName for nm in names: let t = createTypeStub(c, nm) if t != nil: result.add t proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) # The `cg`/`emit` backend stages load the lowered whole-module NIF (transformed # bodies + lifted sigs baked in); the `lower` stage and the frontend (`cmdM`) # read the semchecked `.s.bif`. All module artifacts are binary NIF (`.bif`) # now — one file per stage, no text twin (debug via `tools/bif2nif`). if conf.cmd == cmdNifC and (conf.icBackendStage == "cg" or conf.icBackendStage == "emit"): let t = toGeneratedFile(conf, AbsoluteFile(suffix), ".t.bif").string if fileExists(t): return t result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.bif").string proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym = result = c.syms.getOrDefault(symAsStr)[0] if result != nil: return result let sn = parseSymName(symAsStr) if sn.module.len == 0: return nil # Local symbols shouldn't be hooks let (isBk, realMod) = stripBkSuffix(sn.module) let module = moduleId(c, realMod) # Look up the symbol in the module's index # Try both formats: with module suffix (e.g., "foo.0.modulename") and without (e.g., "foo.0.") # NIF spec allows local symbols to be stored without module suffix var offs = c.mods[module].index.getOrDefault(symAsStr) if offs.offset == 0: # Try the format without module suffix let localKey = sn.name & "." & $sn.count & "." offs = c.mods[module].index.getOrDefault(localKey) if offs.offset == 0: return nil if not alsoConsiderPrivate and offs.vis == Hidden: return nil # Create a stub symbol (skProc: `resolveSym` only resolves hook/routine syms). result = PSym(itemId: c.nextSymId(module, isBk), kindImpl: skProc, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, offs) proc resolveHookSym*(c: var DecodeContext; name: string): PSym = ## Resolves a hook symbol NAME to a PSym. ## Hook symbols are often private (generated =destroy, =wasMoved, etc.) result = resolveSym(c, name, true) proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: FileIndex): PSym = ## Tries to resolve a compiler proc from a module by checking the NIF index. ## Returns nil if the symbol doesn't exist. The NIF disamb is mint order, so ## `name.0.` can be any of the overloads sharing the name — for `newSeq` it ## is the generic magic, not the RTL proc (a refc build then demands codegen ## of the generic and dies on `seq[T]`): enumerate the index entries with ## this basename and pick the one that carries `sfCompilerProc`. result = nil let suffix = moduleSuffix(c.infos.config, moduleFileIdx) let module = moduleId(c, suffix) let prefix = name & "." var candidates: seq[int] = @[] for key in c.mods[module].index.keys: if key.len > prefix.len and key.startsWith(prefix): let sn = parseSymName(key) if sn.name == name: candidates.add sn.count # the loads below can grow `c.mods` (symbols reference other modules), so # resolve only after the index iteration is done for count in candidates: let sym = resolveSym(c, name & "." & $count & "." & suffix, true) if sym != nil: loadSym(c, sym) if sfCompilerProc in sym.flagsImpl: return sym proc loadLogOp(c: var DecodeContext; logOps: var seq[LogEntry]; cur: var Cursor; kind: LogEntryKind; op: TTypeAttachedOp; module: int) = ## Step 2 phase 2: read one `(rep* "key" sym)` from the resident-buffer cursor. cur.into: expect cur, StrLit let key = strVal(cur) skip cur if cur.hasMore and cur.kind == Symbol: let sym = resolveHookSym(c, symName(cur)) if sym != nil: logOps.add LogEntry(kind: kind, op: op, module: module, key: key, sym: sym) # else: symbol not indexed, skip this hook entry skip cur type ModuleSuffix* = distinct string PrecompiledModule* = object topLevel*: PNode # top level statements of the main module deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of logOps*: seq[LogEntry] module*: PSym # set by modulegraphs.nim! reexportedModules*: seq[(string, string)] # (name, suffix) of re-exported MODULE syms; # materialized by modulegraphs.nim genericOffers*: seq[tuple[generic, inst: PSym; concreteTypes: seq[PType]; genericParamsCount: int]] ## generic instances this module created; modulegraphs.nim rebuilds ## `procInstCache` from them so a consumer reuses the instance instead of ## re-instantiating it in its own (operator-blind) module scope. typeOffers*: seq[tuple[generic: PSym; inst: PType]] ## generic TYPE instances this module created; modulegraphs.nim rebuilds ## `typeInstCache` from them so a consumer reuses the baked instance ## (e.g. a `mixin`/`compiles()`-dependent array bound) instead of ## re-instantiating it with a different bound in its own scope. includes*: seq[string] # resolved full paths of files this module `include`s; # replayed into `inclToMod` by modulegraphs.nim so that # nimsuggest can map a query in an include file back to # this module (`parentModule`) and recompile it. proc loadImport(c: var DecodeContext; cur: var Cursor; deps: var seq[ModuleSuffix]) = cur.into: while cur.hasMore and cur.kind == DotToken: skip cur # flags / type if cur.hasMore and cur.kind == StrLit: deps.add ModuleSuffix(strVal(cur)) skip cur else: raiseAssert "expected StrLit but got " & $cur.kind proc loadInclude(c: var DecodeContext; cur: var Cursor; includes: var seq[string]) = ## Reads an `(include . . "path"...)` entry written by `trInclude`. The paths ## are resolved full paths (see semstmts.evalInclude under cmdM/optCompress). cur.into: while cur.hasMore and cur.kind == DotToken: skip cur # flags / type while cur.hasMore and cur.kind == StrLit: includes.add strVal(cur) skip cur proc scanIncludeGraph*(config: ConfigRef): seq[tuple[includer: string; includes: seq[string]]] = ## Standalone "full table" scan of every `.nif` in the nimcache: reads ## only each module's header records — `(modulesrc "path")` (the includer's own ## source) and `(include . . "path"...)` (resolved included files) — and returns ## (includerSource, includedSources) pairs for the modules that `include` ## anything. No `DecodeContext`, no symbol/index loading: it parses the few dep ## tokens at the top of the file and stops at the first non-dep node. ## ## Used by nimsuggest to answer, for a cold-opened *include* file, "which module ## includes me?" without NIF-loading that module — so the includer can be ## *source*-compiled (modules that `include` files are never served from NIF). result = @[] let dir = getNimcacheDir(config) if not dirExists(dir.string): return # The primary module artifacts are `.s.bif` (the sidecars are # `.iface.nif`/`.impl.nif`/`.edges.nif`/`.s.deps.nif`, which this glob excludes). for f in walkFiles((dir / RelativeFile"*.s.bif").string): var m = bif.load(f) var includer = "" var includes: seq[string] = @[] var c = m.buf.beginRead() if c.kind == TagLit and tagIs(c, toNifTag(nkStmtList)): # The dep records (import/include/reexpmod/modulesrc) are written first and # contiguously; `done` short-circuits once the first body node is seen # (`into` forbids an early `break`, so we skip the remainder instead). var done = false c.loopInto: if done or c.kind != TagLit: skip c elif tagIs(c, "include") or tagIs(c, "modulesrc"): let isInc = tagIs(c, "include") var ic = c ic.loopInto: if ic.kind == StrLit: if isInc: includes.add strVal(ic) else: includer = strVal(ic) skip ic skip c elif tagIs(c, "import") or tagIs(c, "reexpmod"): skip c else: done = true skip c if includer.len > 0 and includes.len > 0: result.add (includer, includes) proc nifModuleHasIncludes*(config: ConfigRef; fileIdx: FileIndex): bool = ## Cheap header-only check: does the module's `.nif` contain an ## `(include ...)` record? Used by nimsuggest (`moduleFromNifFile`) to refuse to ## NIF-serve modules that `include` files, so the includer is source-compiled ## and the included symbols never round-trip through NIF (which mishandles their ## owner/line-info on reload). let f = toNifFilename(config, fileIdx) if not fileExists(f): return false var m = bif.load(f) result = false var c = m.buf.beginRead() if c.kind == TagLit and tagIs(c, toNifTag(nkStmtList)): var done = false c.loopInto: if done or c.kind != TagLit: skip c elif tagIs(c, "include"): result = true done = true skip c elif tagIs(c, "modulesrc") or tagIs(c, "import") or tagIs(c, "reexpmod"): skip c else: done = true skip c proc addReexportedEnumFields(c: var DecodeContext; sym: PSym; interf: var TStrTable) = ## When a non-pure enum type is (re-)exported, its fields must also become ## visible (unqualified) to importers. In a from-source build this happens via ## `rawImportSymbol`'s enum handling when the type is imported; the lazy IC ## importer never runs that, so we materialise the fields into the interface ## here, when the export list is processed. loadSym(c, sym) if sym.kindImpl != skType or sfPure in sym.flagsImpl: return let et = sym.typImpl if et == nil: return loadType(c, et) if et.kind notin {tyEnum, tyBool}: return let fields = et.nImpl if fields == nil: return for i in 0 ..< fields.len: let f = fields[i] if f != nil and f.kind == nkSym and f.sym != nil: strTableAdd(interf, f.sym) proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]; interf: var TStrTable; suffix: string; module: int): PrecompiledModule = ## Step 2 phase 2: walk the module body directly over the resident `buf` cursor ## (was a `next(s)` stream walk). `cur` enters at the `(stmts` type dot. Lazy ## loads done here (resolveSym/loadType/…) read INDEPENDENT cursors into the ## resident buffers, never `cur` — so the old export/toffer `jumpTo(saved)` ## save/restore dance is gone. result = PrecompiledModule(topLevel: newNode(nkStmtList)) var localSyms = initTable[string, PSym]() skip cur # the (stmts type dot # Top-level `let`/`var` sections are loaded even without LoadFullAst: they may # declare `{.compileTime.}` globals whose VM slots the importer initializes # eagerly (pipelines.initLoadedCompileTimeGlobals), which needs them visible in # `topLevel`. They sit in the module header before `(implementation)`. var cont = true while cont and cur.hasMore: if cur.kind != TagLit: cont = false else: if tagIs(cur, "replay"): # Always load replay actions (macro cache operations) cur.into: while cur.hasMore: let replayNode = loadNode(c, cur, suffix, localSyms) if replayNode != nil: result.topLevel.sons.add replayNode elif tagIs(cur, "unusedid"): # backend id seed — consumed eagerly by `moduleId`/`readUnusedId`; just # skip past it here so the rest of the header still loads. skip cur elif tagIs(cur, "repconverter"): loadLogOp(c, result.logOps, cur, ConverterEntry, attachedTrace, module) elif tagIs(cur, "repdestroy"): loadLogOp(c, result.logOps, cur, HookEntry, attachedDestructor, module) elif tagIs(cur, "repwasmoved"): loadLogOp(c, result.logOps, cur, HookEntry, attachedWasMoved, module) elif tagIs(cur, "repcopy"): loadLogOp(c, result.logOps, cur, HookEntry, attachedAsgn, module) elif tagIs(cur, "repsink"): loadLogOp(c, result.logOps, cur, HookEntry, attachedSink, module) elif tagIs(cur, "repdup"): loadLogOp(c, result.logOps, cur, HookEntry, attachedDup, module) elif tagIs(cur, "reptrace"): loadLogOp(c, result.logOps, cur, HookEntry, attachedTrace, module) elif tagIs(cur, "repdeepcopy"): loadLogOp(c, result.logOps, cur, HookEntry, attachedDeepCopy, module) elif tagIs(cur, "repenumtostr"): loadLogOp(c, result.logOps, cur, EnumToStrEntry, attachedTrace, module) elif tagIs(cur, "repmethod"): loadLogOp(c, result.logOps, cur, MethodEntry, attachedTrace, module) elif tagIs(cur, "reppureenum"): loadLogOp(c, result.logOps, cur, PureEnumEntry, attachedTrace, module) elif tagIs(cur, "export"): cur.into: while cur.hasMore and cur.kind == DotToken: skip cur # flags / type while cur.hasMore: if cur.kind == Symbol: let symAsStr = symName(cur) # Skip symbols re-exported by this dependency but owned by the module # being compiled fresh (they would collide with the fresh originals). if c.mainModuleSuffix.len == 0 or parseSymName(symAsStr).module != c.mainModuleSuffix: let sym = resolveSym(c, symAsStr, false) if sym != nil: strTableAdd(interf, sym) addReexportedEnumFields(c, sym, interf) skip cur else: raiseAssert "expected Symbol or ParRi but got " & $cur.kind & " in export list of module " & suffix elif tagIs(cur, "include"): loadInclude(c, cur, result.includes) elif tagIs(cur, "import"): loadImport(c, cur, result.deps) elif tagIs(cur, "reexpmod"): # a re-exported MODULE: (reexpmod "name" "suffix"); the module sym is a # qualifier in this module's interface — materialized by modulegraphs. var mname, msuffix = "" cur.into: if cur.hasMore and cur.kind == StrLit: (mname = strVal(cur); skip cur) if cur.hasMore and cur.kind == StrLit: (msuffix = strVal(cur); skip cur) if mname.len > 0 and msuffix.len > 0: result.reexportedModules.add (mname, msuffix) elif tagIs(cur, "offer"): # (offer ...) — resolve # to PSyms/PTypes; modulegraphs registers them into `procInstCache`. # Best-effort: a type that fails to resolve drops the whole offer. var genSym, instSym: PSym = nil var paramsCount = 0 var cts: seq[PType] = @[] var idx = 0 var ok = true cur.into: while cur.hasMore: if cur.kind == Symbol: if idx == 0: genSym = resolveHookSym(c, symName(cur)) elif idx == 1: instSym = resolveHookSym(c, symName(cur)) else: let ct = tryCreateTypeStub(c, symName(cur)) if ct == nil: ok = false else: cts.add ct inc idx skip cur elif cur.kind == IntLit: paramsCount = int(intVal(cur)) skip cur else: skip cur if ok and genSym != nil and instSym != nil: result.genericOffers.add (genSym, instSym, cts, paramsCount) elif tagIs(cur, "toffer"): # (toffer "" "") — intern the two full names, # resolve, FULLY load the instance (so `searchInstTypes` can match its # params). Best-effort: a failure to resolve drops the offer. var genName, instName = "" var idx = 0 cur.into: while cur.hasMore: if cur.kind == StrLit: if idx == 0: genName = strVal(cur) elif idx == 1: instName = strVal(cur) inc idx skip cur else: skip cur if genName.len > 0 and instName.len > 0: let genSym = resolveHookSym(c, genName) let inst = tryCreateTypeStub(c, instName) if genSym != nil and inst != nil: loadType(c, inst) result.typeOffers.add (genSym, inst) elif tagIs(cur, "modulesrc"): # self-identification record for the standalone include-graph scanner; # not needed by the loader, just skip past it. skip cur elif tagIs(cur, "expansion"): # template/macro expansion usage record for tooling (`idetools` scans it # as a `Symbol` use); the loader itself needs nothing from it. skip cur elif tagIs(cur, "sig"): # signature-symbol occurrence record for tooling (`idetools` scans it as a # `Symbol` use); the loader itself needs nothing from it. skip cur elif tagIs(cur, "implementation"): cont = false elif LoadFullAst in flags or tagIs(cur, toNifTag(nkLetSection)) or tagIs(cur, toNifTag(nkVarSection)) or tagIs(cur, toNifTag(nkPragma)): # Parse the full statement. let/var sections are loaded unconditionally # (see above) so `{.compileTime.}` globals reach the eager initializer. # Top-level pragmas are loaded too: a module-level `{.emit.}` (and the # `{.push/pop.}` around it) must reach the `cg` stage's genPragma/genEmit, # else e.g. a `#include` is dropped and the generated C won't compile. # writeToplevelNode routes these into this header section. let stmtNode = loadNode(c, cur, suffix, localSyms) if stmtNode != nil: result.topLevel.sons.add stmtNode else: cont = false proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHidden: var TStrTable; flags: set[LoadFlag] = {}): PrecompiledModule = # Ensure module index is loaded - moduleId returns the FileIndex for this suffix let module = moduleId(c, string(suffix), flags) # Load the module AST (or just replay actions if loadFullAst is false). # processTopLevel also collects export instructions. Step 2 phase 2: read the # body straight from the resident `buf` cursor (no stream, no rewind — lazy # loads use independent cursors so they never disturb this one). var cur = beginRead(c.mods[module].buf) if cur.kind == TagLit and tagIs(cur, toNifTag(nkStmtList)): inc cur # enter (stmts (past the tag head, onto the flags dot) skip cur # flags dot (processTopLevel skips the type dot itself) result = processTopLevel(c, cur, flags, interf, string(suffix), module.int) else: result = PrecompiledModule(topLevel: newNode(nkStmtList)) # Populate interface tables from the NIF index structure # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym # Use exports collected by processTopLevel populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix)) proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; flags: set[LoadFlag] = {}): PrecompiledModule = let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f)) result = loadNifModule(c, suffix, interf, interfHidden, flags) proc writeLoweredModule*(c: var DecodeContext; config: ConfigRef; precomp: PrecompiledModule; hooks: openArray[LogEntry]; outfile: string) = ## Re-serialize a backend-loaded module as a FULL module NIF (`.t.nif`) whose ## routine `(sd)` entries carry their TRANSFORMED bodies (the `lower` stage set ## them, recursively lifting nested closures — including the async state-machine ## procs whose inner closure the per-`(lowered)`-entry path failed to cross) and ## whose lambda-lift-minted entities (closure-env types/syms, lifted nested ## procs) are real, indexed defs. The `cg` stage then loads it through the ## normal module loader (`moduleFromNifFile`), so a transformed body arrives via ## `loadSymFromCursor`'s Step-A 2-way-body slot WITH the lifted signature — no ## `(lowered)` side-car, no `:envP` re-weld. This realizes `ic_ideas.md`'s eager ## two-way body whole-module. let thisModule = precomp.module.positionImpl.int32 # Routines → Sealed (cross-routine refs become SymUse, defs emitted once below); # types/globals/params/locals stay Complete and emit real defs (the `.t.nif` is # the sole source the cg stage reads — no `.s.nif` fallback for them). sealLoadedRoutines(c) var w = Writer(infos: newLineInfoWriter(config), currentModule: thisModule) w.deps = newIcBuilder(64) w.inProc = 1 w.lowering = true var content = newIcBuilder(300) let rootInfo = trLineInfo(w, precomp.topLevel.info) createStmtList(content, rootInfo) # This module's ops (hooks/converters/methods/pure-enums) loaded from `.s.nif`, # plus the type-bound ops the lower transform just lifted (closure-env # `=destroy` etc., which have no `.s.nif` entry). for op in precomp.logOps: if op.module == thisModule.int: writeOp(w, content, op) for op in hooks: writeOp(w, content, op) var bottom = newIcBuilder(300) # Imperative init code + global let/var/const sections + replay actions — all # that a backend-loaded `topLevel` carries (routines are lazy index sdefs, not # here). Emits + seals the module's globals. w.writeToplevelNode content, bottom, precomp.topLevel # TYPE DEFS: emit into the `.t.bif` ONLY the owned types the lower stage actually # loaded (`Complete`) — those are the ones it can have MUTATED (proc type → # `ccClosure`, env type grown with captured fields), so the `.t.bif` must carry # the mutated version. Every untouched owned type stays `Partial` → a reference to # it below writes a `SymUse` that a cg/emit consumer resolves from the `.s.bif` # (loader fallback `typeCursor`/`ensureSemBuf`), so we no longer force-load + # re-serialize the whole type table here. Guard on `Complete`: a type reached as # an owned son of an earlier def is already `Sealed` (emitted inline) — skip it. for t in loadedModuleTypes(c, FileIndex thisModule): if t.state == Complete: writeType(w, bottom, t) # Routine DEFS (with transformed bodies) + CONST DEFS this module owns, sourced # from the index. Consts are lazy index sdefs too (like routines): the # backend-loaded `topLevel` carries only runtime init (module var/let sections), # NOT consts — especially `importc`/magic consts (`SIG_DFL`, `hasAllocStack`, …) # which have no runtime init at all. `writeNifModule` emitted them via the full # AST walk; here we must enumerate them from the index, else a cross-module # `SymUse` resolves to a nil `skModule` stub (`expr(skModule); unknown symbol`). for s in moduleSymbolStubs(c, FileIndex thisModule): if (s.kindImpl in routineKinds or s.kindImpl == skConst) and s.itemId.module == thisModule: writeSymDef(w, bottom, s) # Lifted hook ROUTINES (`@bk`, NEW in the lower stage — no `.s.nif` sdef, so # absent from `moduleSymbolStubs`): emit each as a full def (sig + transformed # body) so `injectDestructorCalls` in cg resolves the loaded env's `=destroy`. var emittedHooks = initHashSet[int32]() for op in hooks: if op.sym != nil and op.sym.kindImpl in routineKinds and not emittedHooks.containsOrIncl(op.sym.itemId.item): writeSymDef(w, bottom, op.sym) # deps / reexports / offers — mirror writeNifModule so the cg backend closure # walk, interface re-export and generic-instance reuse all work off `.t.nif`. for dep in precomp.deps: if not w.depSuffixes.containsOrIncl(dep.string): w.deps.addParLe importTag, NoLineInfo w.deps.addDotToken w.deps.addDotToken w.deps.addStrLit dep.string w.deps.addParRi for (mname, msuffix) in precomp.reexportedModules: w.deps.addParLe reexpModTag, NoLineInfo w.deps.addStrLit mname w.deps.addStrLit msuffix w.deps.addParRi for off in precomp.genericOffers: w.deps.addParLe offerTag, NoLineInfo w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.generic)), NoLineInfo w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.inst)), NoLineInfo w.deps.addIntLit off.genericParamsCount for ct in off.concreteTypes: w.deps.addSymUse pool.syms.getOrIncl(typeToNifSym(ct, w.infos.config)), NoLineInfo w.deps.addParRi for off in precomp.typeOffers: w.deps.addParLe typeOfferTag, NoLineInfo w.deps.addStrLit w.toNifSymName(off.generic) w.deps.addStrLit typeToNifSym(off.inst, w.infos.config) w.deps.addParRi # OWNER MUST EMIT offered types this module owns (see writeNifModule). for off in precomp.genericOffers: for ct in off.concreteTypes: if ct != nil and ct.uniqueId.module == w.currentModule and ct.state == Complete: writeType(w, bottom, ct) for off in precomp.typeOffers: if off.inst != nil and off.inst.uniqueId.module == w.currentModule and off.inst.state == Complete: writeType(w, bottom, off.inst) # Assemble exactly as writeNifModule: (stmts . . # (implementation) ). content.addParLe implTag, NoLineInfo content.addParRi() addAll(content, bottom) content.addParRi() var dest = newIcBuilder(600) createStmtList(dest, rootInfo) # Carry the seed FORWARD: the lower stage minted backend syms/types from the # per-module counters (seeded out of the `.s.bif`'s `(unusedid)`), so they now # hold the post-lower high-water mark. Record it so the `cg` stage — which # loads THIS `.t.bif` and mints still more (RTTI hooks) — seeds above it too. let lfi = FileIndex thisModule let loweredSeed = if c.mods.hasKey(lfi): max(c.mods[lfi].symCounter, c.mods[lfi].typeCounter) else: 0'i32 dest.addParLe unusedIdTag, NoLineInfo dest.addIntLit loweredSeed.int64 dest.addParRi() addAll(dest, w.deps) addStmtsBody(dest, content) dest.addParRi() # Step 3: the lowered whole-module artifact is binary NIF (`.t.bif`) too — the # cg/emit stages load it via `toNifFilename`. No text twin (debug via bif2nif). storeBif(dest, outfile, "." & extractModuleSuffix(outfile)) when isMainModule: import std / syncio let obj = parseSymName("a.123.sys") echo obj.name, " ", obj.module, " ", obj.count let objb = parseSymName("abcdef.0121") echo objb.name, " ", objb.module, " ", objb.count