From 7171e6f01f846a511a5fad8d1ab24baaee66e308 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 14 Jun 2026 22:35:06 +0200 Subject: [PATCH] IC: progress (#25879) --- compiler/ast.nim | 86 ++- compiler/ast2nif.nim | 853 ++++++++++++++++++++++++--- compiler/astdef.nim | 20 +- compiler/ccgcalls.nim | 12 +- compiler/ccgexprs.nim | 34 +- compiler/ccgstmts.nim | 7 +- compiler/ccgtypes.nim | 127 +++- compiler/ccgutils.nim | 7 +- compiler/cgen.nim | 275 ++++++++- compiler/cgendata.nim | 17 + compiler/cgmeth.nim | 2 + compiler/cnif.nim | 726 +++++++++++++++++++++++ compiler/commands.nim | 52 +- compiler/deps.nim | 829 ++++++++++++++++++++++---- compiler/enumtostr.nim | 1 + compiler/ic/replayer.nim | 34 ++ compiler/icconfig.nim | 127 ++++ compiler/itemids.nim | 98 +++ compiler/lambdalifting.nim | 18 +- compiler/liftdestructors.nim | 22 +- compiler/lookups.nim | 3 + compiler/lowerings.nim | 6 +- compiler/main.nim | 14 +- compiler/mangleutils.nim | 28 +- compiler/modulegraphs.nim | 369 +++++++++++- compiler/modules.nim | 2 +- compiler/msgs.nim | 3 + compiler/nifbackend.nim | 479 +++++++++++++-- compiler/nim.nim | 9 +- compiler/nimconf.nim | 8 +- compiler/options.nim | 66 +++ compiler/parser.nim | 5 +- compiler/pipelines.nim | 61 +- compiler/pipelineutils.nim | 2 +- compiler/semcall.nim | 19 +- compiler/semdata.nim | 18 +- compiler/semexprs.nim | 58 ++ compiler/semfold.nim | 13 +- compiler/seminst.nim | 59 +- compiler/semstmts.nim | 8 + compiler/semtypes.nim | 18 +- compiler/semtypinst.nim | 55 +- compiler/sighashes.nim | 61 ++ compiler/sigmatch.nim | 13 +- compiler/typekeys.nim | 138 ++++- compiler/vm.nim | 6 +- compiler/vmgen.nim | 10 +- compiler/vmops.nim | 4 +- doc/ic.md | 412 +++++++++---- koch.nim | 55 +- lib/system/mmdisp.nim | 5 +- tests/codegen/titaniummangle_nim.nim | 2 +- tests/generics/mopensymdot.nim | 18 + tests/generics/topensymdot.nim | 12 + tests/js/tcodegendeclproc.nim | 2 +- 55 files changed, 4843 insertions(+), 545 deletions(-) create mode 100644 compiler/cnif.nim create mode 100644 compiler/icconfig.nim create mode 100644 compiler/itemids.nim create mode 100644 tests/generics/mopensymdot.nim create mode 100644 tests/generics/topensymdot.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index b04a102b20..068b3c4f86 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -36,6 +36,13 @@ proc setupProgram*(config: ConfigRef; cache: IdentCache) = when not defined(nimKochBootstrap): program = createDecodeContext(config, cache) +proc setIcMainModule*(fileIdx: FileIndex) = + ## Tells the IC loader which module is being compiled fresh, so that + ## re-exports of that module's symbols by dependencies are not loaded as + ## duplicate stubs. + when not defined(nimKochBootstrap): + ast2nif.setMainModule(program, fileIdx) + template loadSym(s: PSym) = ## Loads a symbol from NIF file if it's in Partial state. when not defined(nimKochBootstrap): @@ -70,6 +77,16 @@ proc backendEnsureMutable*(t: PType) {.inline.} = # ^ IC review this later if t.state == Partial: loadType(t) +proc unsealForTransform*(t: PType) {.inline.} = + ## The transformer/lambda lifting also run inside `nim m` when the VM + ## compiles a LOADED routine (macro evaluation, `getImpl`). Their mutations + ## are process-local — transformed bodies are never written back to a NIF — + ## so downgrade the loaded type to mutable, mirroring the `cmdNifC` loader + ## which loads everything `Complete` for exactly this reason (see + ## `ast2nif.loadedState`). + if t.state == Partial: loadType(t) + if t.state == Sealed: t.state = Complete + proc owner*(s: PSym): PSym {.inline.} = if s.state == Partial: loadSym(s) result = s.ownerFieldImpl @@ -221,7 +238,10 @@ proc position*(s: PSym): int {.inline.} = result = s.positionImpl proc `position=`*(s: PSym, val: int) {.inline.} = - assert s.state != Sealed + # No `Sealed` guard: the VM reuses `position` as a register slot while compiling + # a macro for execution (see `vmgen.genGenericParams`), which under IC may be a + # macro loaded from a NIF file. The macro is run, not code-generated, so this + # scratch mutation is harmless. if s.state == Partial: loadSym(s) s.positionImpl = val @@ -445,9 +465,13 @@ var gconfig {.threadvar.}: Gconfig proc setUseIc*(useIc: bool) = gconfig.useIc = useIc proc comment*(n: PNode): string = - if nfHasComment in n.flags and not gconfig.useIc: - # IC doesn't track comments, see `packed_ast`, so this could fail - result = gconfig.comments[n.nodeId] + if nfHasComment in n.flags: + # NIF-based IC doesn't serialize comments, but the comment table is keyed by + # the node's address (`nodeId`), which is unique among live nodes; a loaded + # node that carries `nfHasComment` simply has no entry here (its comment was + # set in another process), so `getOrDefault` safely returns "" for it while + # in-process VM macro nodes (e.g. newCommentStmtNode) still round-trip. + result = gconfig.comments.getOrDefault(n.nodeId) else: result = "" @@ -478,13 +502,6 @@ proc getPIdent*(a: PNode): PIdent {.inline.} = of nkOpenSymChoice, nkClosedSymChoice, nkOpenSym: a.sons[0].sym.name else: nil -const - moduleShift = when defined(cpu32): 20 else: 24 - -template toId*(a: ItemId): int = - let x = a - (x.module.int shl moduleShift) + x.item.int - template id*(a: PType | PSym): int = toId(a.itemId) type @@ -493,28 +510,44 @@ type symId*: int32 typeId*: int32 sealed*: bool + backendMinted*: bool disambTable*: CountTable[PIdent] -const - PackageModuleId* = -3'i32 - proc idGeneratorFromModule*(m: PSym): IdGenerator = assert m.kind == skModule result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0, disambTable: initCountTable[PIdent]()) result.disambTable.inc m.name +proc idGeneratorForBackend*(m: PSym): IdGenerator = + ## Like `idGeneratorFromModule`, but for IC codegen (`nim nifc`): symbols and + ## types minted fresh during codegen (transf labels/temps, lifted hooks, type + ## copies) must not collide with the itemIds the NIF loader synthesizes for + ## lazily-loaded symbols/types of the same module — those come from a + ## per-module load-order counter that keeps running while codegen mints its + ## own ids. A collision corrupts itemId-keyed tables, e.g. `transf`'s inline + ## iterator mapping then substitutes a random loaded sym (a call's callee) + ## with a `:tmp` block label. Backend-minted ids carry a marker bit in the + ## module half (see `itemids.backendItemId`), so the two id spaces are + ## disjoint by construction. + assert m.kind == skModule + result = IdGenerator(module: m.itemId.module, symId: 0, typeId: 0, + backendMinted: true, disambTable: initCountTable[PIdent]()) + result.disambTable.inc m.name + proc idGeneratorForPackage*(nextIdWillBe: int32): IdGenerator = result = IdGenerator(module: PackageModuleId, symId: nextIdWillBe - 1'i32, typeId: 0, disambTable: initCountTable[PIdent]()) proc nextSymId(x: IdGenerator): ItemId {.inline.} = assert(not x.sealed) inc x.symId - result = ItemId(module: x.module, item: x.symId) + result = if x.backendMinted: backendItemId(x.module, x.symId) + else: itemId(x.module, x.symId) proc nextTypeId*(x: IdGenerator): ItemId {.inline.} = assert(not x.sealed) inc x.typeId - result = ItemId(module: x.module, item: x.typeId) + result = if x.backendMinted: backendItemId(x.module, x.typeId) + else: itemId(x.module, x.typeId) when false: proc nextId*(x: IdGenerator): ItemId {.inline.} = @@ -1043,6 +1076,11 @@ proc newType*(kind: TTypeKind; idgen: IdGenerator; owner: PSym; son: sink PType if result.itemId.module == 55 and result.itemId.item == 2: echo "KNID ", kind writeStackTrace() + when defined(icDbg): + if kind == tyOpenArray: + echo "NEWTYPE openArray id=", id.module, ".", id.item, + " owner=", (if owner != nil: owner.name.s else: "nil") + echo getStackTrace() proc setSons*(dest: PType; sons: sink seq[PType]) {.inline.} = assert dest.kind != tyProc or sons.len <= 1 @@ -1105,10 +1143,19 @@ proc copyType*(t: PType, idgen: IdGenerator, owner: PSym): PType = assignType(result, t) result.symImpl = t.sym # backend-info should not be copied -proc exactReplica*(t: PType): PType = +proc exactReplica*(t: PType; idgen: IdGenerator): PType = + ## Replica that KEEPS `itemId` — the generic-param binding tables + ## (`LayeredIdTable`) key on it, so the copy must keep matching its + ## original — but mints a FRESH `uniqueId`: uniqueId is the SERIALIZATION + ## identity (NIF type names key on it) and must be unique per instance. + ## Replicas sharing the original's uniqueId serialized as duplicate defs + ## under one NIF name; the loader collapsed them into a single type, + ## losing their flag differences (use-site `tfUnresolved` typedescs) or + ## their structure (meta instance bodies shadowing a generic's canonical + ## body). result = PType(kind: t.kind, ownerFieldImpl: t.owner, sizeImpl: defaultSize, alignImpl: defaultAlignment, itemId: t.itemId, - uniqueId: t.uniqueId) + uniqueId: nextTypeId(idgen)) assignType(result, t) result.symImpl = t.sym # backend-info should not be copied @@ -1271,6 +1318,9 @@ proc transitionNoneToSym*(n: PNode) = transitionNodeKindCommon(nkSym) template transitionSymKindCommon*(k: TSymKind) = + # Under IC the symbol may still be an unloaded stub (`skStub`); materialise it + # first so its kind-specific fields (read below as `obj.*`) actually exist. + if s.state == Partial: loadSym(s) let obj {.inject.} = s[] s[] = TSym(kindImpl: k, itemId: obj.itemId, magicImpl: obj.magicImpl, typImpl: obj.typImpl, name: obj.name, infoImpl: obj.infoImpl, ownerFieldImpl: obj.ownerFieldImpl, flagsImpl: obj.flagsImpl, astImpl: obj.astImpl, diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 8847af32b7..c4bab071ee 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -10,8 +10,11 @@ ## AST to NIF bridge. import std / [assertions, tables, sets] -from std / strutils import startsWith +from std / strutils import startsWith, endsWith, contains from std / os import fileExists +from std / syncio import readFile +from std / algorithm import sort +import "../dist/checksums/src/checksums" / sha1 import astdef, idents, msgs, options import lineinfos as astli import pathutils #, modulegraphs @@ -23,6 +26,9 @@ import typekeys import ic / [enum2nif] proc typeToNifSym(typ: PType; config: ConfigRef): string = + # NOTE: uniqueId is the serialization identity and is unique per instance — + # `exactReplica` keeps only itemId shared with its original (see ast.nim) + assert not typ.uniqueId.isBackendMinted result = "`t" result.addInt ord(typ.kind) result.add '.' @@ -30,6 +36,16 @@ proc typeToNifSym(typ: PType; config: ConfigRef): string = result.add '.' result.add modname(typ.uniqueId.module, config) +proc icNifTypeName*(typ: PType; config: ConfigRef): string = + ## The serialized NIF name of a type, recorded next to RTTI data + ## definitions in the cnif artifact so a later run can re-demand the + ## typeinfo when a reused TU still references it (the def-retention + ## check). Backend-minted types have no NIF name. + if typ != nil and not typ.uniqueId.isBackendMinted: + result = typeToNifSym(typ, config) + else: + result = "" + proc toHookIndexEntry*(config: ConfigRef; typeId: ItemId; hookSym: PSym): HookIndexEntry = ## Converts a type ItemId and hook symbol to a HookIndexEntry for the NIF index. let typeSymName = "`t" & $typeId.item & "." & cachedModuleSuffix(config, typeId.module.FileIndex) @@ -144,7 +160,7 @@ const symDefTagName = "sd" typeDefTagName = "td" -let +var sdefTag = registerTag(symDefTagName) tdefTag = registerTag(typeDefTagName) hiddenTypeTag = registerTag(hiddenTypeTagName) @@ -161,19 +177,37 @@ type #writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later writtenPackages: HashSet[string] -const - # Symbol kinds that are always local to a proc and should never have module suffix - skLocalSymKinds = {skParam, skForVar, skResult, skTemp} - proc isLocalSym(sym: PSym): bool {.inline.} = - sym.kindImpl in skLocalSymKinds or - (sym.kindImpl in {skVar, skLet} and {sfGlobal, sfThread} * sym.flagsImpl == {} and - (sym.ownerFieldImpl == nil or sym.ownerFieldImpl.kindImpl != skModule)) + ## Every symbol is emitted as a *global* (module-suffixed) name so that its + ## `sdef` gets an index entry and is resolvable by index lookup even when + ## referenced from a different index entry than the one that physically + ## contains the definition. This matters for symbols shared across entries: + ## generic params of a forward declaration vs its implementation, and proc-type + ## params shared between an enclosing proc and a nested object's proc-type + ## field. The per-module `disamb` counter keeps `name.disamb.module` unique, so + ## globalising cannot cause clashes. This trades index size for correctness; + ## size/speed can be optimised later. + false + +const + PkgMarker = "`pkg" + ## Appended to the ident of `skPackage` symbols in NIF names. A package sym + ## has no module of its own: it is written once into every module NIF that + ## references it, named with that module's suffix and its own (independent) + ## disamb counter. Without the marker it can collide with a module-level + ## symbol of the same name and disamb — e.g. extccomp's `compiler` template + ## vs the `compiler` package — and the module sym's owner then resolves to + ## the wrong symbol on load, producing a cyclic owner chain that hangs every + ## owner-walk (sighashes.hashSym etc.). Backtick cannot appear in a Nim + ## identifier, mirroring the "`t" namespace used by `typeToNifSym`. proc toNifSymName(w: var Writer; sym: PSym): string = ## Generate NIF name for a symbol: local names are `ident.disamb`, ## global names are `ident.disamb.moduleSuffix` + assert not sym.itemId.isBackendMinted result = sym.name.s + if sym.kindImpl == skPackage: + result.add PkgMarker result.add '.' result.addInt sym.disamb if not isLocalSym(sym) and sym.itemId notin w.locals: @@ -183,8 +217,11 @@ proc toNifSymName(w: var Writer; sym: PSym): string = result.add modname(module, w.infos.config) -proc globalName(sym: PSym; config: ConfigRef): string = +proc globalName*(sym: PSym; config: ConfigRef): string = result = sym.name.s + if sym.kindImpl == skPackage: + # stubs store the clean name; the NIF index is keyed by the marked one + result.add PkgMarker result.add '.' result.addInt sym.disamb result.add '.' @@ -221,6 +258,18 @@ proc parseSymName*(s: string): ParsedSymName = dec i return ParsedSymName(name: s, module: "") +proc stubKindAndName(cache: IdentCache; rawName: string): (TSymKind, PIdent) = + ## The user-visible name of a symbol stub must NOT keep NIF-only name + ## decorations: the `PkgMarker` of package symbols would otherwise leak into + ## every reader of `name.s` that runs before the stub is fully loaded + ## (e.g. vmgen's callback keys built from owner chains). The marker also + ## tells us the symbol kind up front, which `globalName` uses to rebuild + ## the marked NIF name for the index lookup. + if rawName.endsWith(PkgMarker): + (skPackage, cache.getIdent(rawName[0 ..< rawName.len - PkgMarker.len])) + else: + (skStub, cache.getIdent(rawName)) + template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = dest.addParLe tag body @@ -262,6 +311,15 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = dest.addIntLit typ.alignImpl dest.addIntLit typ.paddingAtEndImpl dest.addIntLit typ.itemId.item # nonUniqueId + # `exactReplica` keeps the canonical type's itemId (binding-table key) + # while minting a fresh uniqueId (the NIF name): when the two halves + # name different modules, the loader cannot reconstruct itemId.module + # from the type's name — serialize it explicitly + if typ.itemId.module != typ.uniqueId.module and + not typ.itemId.isBackendMinted: + dest.addStrLit modname(typ.itemId.module, w.infos.config) + else: + dest.addDotToken writeType(w, dest, typ.typeInstImpl) #if typ.kind in {tyProc, tyIterator} and typ.nImpl != nil and typ.nImpl.kind != nkFormalParams: @@ -281,7 +339,14 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = if typ == nil: dest.addDotToken() - elif typ.itemId.module == w.currentModule and typ.state == Complete: + elif typ.uniqueId.module == w.currentModule and typ.state == Complete: + # Ownership for serialization is decided by `uniqueId`, not `itemId`: the NIF + # name (`typeToNifSym`) and the loader (`createTypeStub`) both key off + # `uniqueId`, so the module that *created* the type (uniqueId.module) must be + # the one that emits its definition. `itemId.module` can be reassigned and + # diverge from `uniqueId.module`; gating on it filed the def in the wrong + # module (or nowhere), leaving dangling references (e.g. `symbol has no + # offset` for a `pointer` type whose itemId.module drifted away). typ.state = Sealed writeTypeDef(w, dest, typ) else: @@ -301,28 +366,19 @@ proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) = dest.addStrLit lib.name writeNode w, dest, lib.path -proc collectGenericParams(w: var Writer; n: PNode) = - ## Pre-collect generic param symbols into w.locals before writing the type. - ## This ensures generic params get consistent short names, and their sdefs - ## are written in the type (where lazy loading can find them). - if n == nil: return - case n.kind - of nkSym: - if n.sym != nil and w.inProc > 0: - w.locals.incl(n.sym.itemId) - of nkIdentDefs, nkVarTuple: - for i in 0 ..< max(0, n.len - 2): - collectGenericParams(w, n[i]) - of nkGenericParams: - for child in n: - collectGenericParams(w, child) - else: - discard - proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addParLe sdefTag, trLineInfo(w, sym.infoImpl) dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo - if {sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}: + # The `x` marker means "importable as a bare identifier into an importer's + # scope". Object fields carry `sfExported` (so they are visible via `obj.field` + # across modules) but must NOT become bare-importable: otherwise an exported + # field name (e.g. `HSlice.a`, whose type is a generic param `T`) leaks into + # module scope and a template's open/mixin symbol of the same name resolves to + # the field instead of a local, producing "type mismatch: got 'T'". Fields are + # still indexed (for `obj.field` resolution via the loaded object type); they + # are merely not advertised as importable. `skEnumField` stays importable — + # enum values are legitimately usable as bare identifiers. + if sym.kindImpl != skField and {sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}: dest.addIdent "x" else: dest.addDotToken @@ -353,14 +409,12 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeLib(w, dest, sym.annexImpl) - # For routine symbols, pre-collect generic params into w.locals before writing - # the type. This ensures they get consistent short names, and their sdefs are - # written in the type where lazy loading can find them via extractLocalSymsFromTree. - if sym.kindImpl in routineKinds and sym.astImpl != nil and sym.astImpl.len > genericParamsPos: - inc w.inProc - collectGenericParams(w, sym.astImpl[genericParamsPos]) - dec w.inProc - + # Generic params are written as *global* symbols (with a module suffix) so that + # they get their own index entries and can be looked up lazily. This matters for + # generic routines that have a separate forward declaration and implementation: + # the two share the same generic param symbols, but each is serialized as its own + # index entry. If the params were local, a reference from the implementation's + # entry could not resolve the sdef emitted in the forward declaration's entry. writeType(w, dest, sym.typImpl) writeSym(w, dest, sym.ownerFieldImpl) # Store the AST for routine symbols and constants @@ -403,11 +457,24 @@ proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = if sym == nil: dest.addDotToken() - elif shouldWriteSymDef(w, sym): + return + # Compare lazy-aware, not the raw field: a sym node loaded from a NIF carries + # `typField == nil` plus `nfLazyType`, meaning "my type is the symbol's + # type". Comparing `typField` directly would re-serialize such a node as + # `(ht . sym)` — an explicitly nil node type — and the next loader gets a + # nil-typed node *without* the lazy fallback (semfold & friends crash on + # `n.typ == nil`). Only a genuinely nil node type keeps the explicit form. + # (ast.nim's `typ` accessor is not importable here; replicate its fallback. + # For a still-Partial sym `typImpl` is nil, which also compares equal below + # and yields the plain SymUse form — exactly the lazy round-trip we want.) + var nodeTyp = n.typField + if nodeTyp == nil and nfLazyType in n.flags: + nodeTyp = sym.typImpl + if shouldWriteSymDef(w, sym): sym.state = Sealed - if n.typField != n.sym.typImpl: + if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): - writeType(w, dest, n.typField) + writeType(w, dest, nodeTyp) writeSymDef(w, dest, sym) else: writeSymDef(w, dest, sym) @@ -415,9 +482,9 @@ proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! let info = trLineInfo(w, n.info) - if n.typField != n.sym.typImpl: + if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, info: - writeType(w, dest, n.typField) + writeType(w, dest, nodeTyp) dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info else: dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info @@ -433,9 +500,10 @@ template withNode(w: var Writer; dest: var TokenBuf; n: PNode; body: untyped) = dest.addParRi proc addLocalSym(w: var Writer; n: PNode) = - ## Add symbol from a node to locals set if it's a symbol node - if n != nil and n.kind == nkSym and n.sym != nil and w.inProc > 0: - w.locals.incl(n.sym.itemId) + ## Previously forced proc-local symbols to be written without a module suffix. + ## All symbols are now emitted as global (see `isLocalSym`), so `w.locals` is + ## intentionally left empty. + discard proc addLocalSyms(w: var Writer; n: PNode) = case n.kind @@ -467,12 +535,14 @@ proc moduleSuffix(conf: ConfigRef; f: FileIndex): string = proc trImport(w: var Writer; n: PNode) = for child in n: - if child.kind == nkSym: + if child.kind == nkSym and child.sym.kindImpl == skModule: + # a non-module sym appears for an `import v` inside an unexpanded + # template body (e.g. stew/importops' `when compiles((; import v))`): + # not a dependency edge, the import resolves at the expansion site w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) w.deps.addDotToken # flags w.deps.addDotToken # type let s = child.sym - assert s.kindImpl == skModule let fp = moduleSuffix(w.infos.config, s.positionImpl.FileIndex) w.deps.addStrLit fp # raw string literal, no wrapper needed w.deps.addParRi @@ -495,21 +565,51 @@ proc trExport(w: var Writer; n: PNode) = w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(s)), NoLineInfo w.deps.addParRi -let replayTag = registerTag("replay") -let repConverterTag = registerTag("repconverter") -let repDestroyTag = registerTag("repdestroy") -let repWasMovedTag = registerTag("repwasmoved") -let repCopyTag = registerTag("repcopy") -let repSinkTag = registerTag("repsink") -let repDupTag = registerTag("repdup") -let repTraceTag = registerTag("reptrace") -let repDeepCopyTag = registerTag("repdeepcopy") -let repEnumToStrTag = registerTag("repenumtostr") -let repMethodTag = registerTag("repmethod") -#let repClassTag = registerTag("repclass") -let includeTag = registerTag("include") -let importTag = registerTag("import") -let implTag = registerTag("implementation") +var replayTag = registerTag("replay") +var repConverterTag = registerTag("repconverter") +var repDestroyTag = registerTag("repdestroy") +var repWasMovedTag = registerTag("repwasmoved") +var repCopyTag = registerTag("repcopy") +var repSinkTag = registerTag("repsink") +var repDupTag = registerTag("repdup") +var repTraceTag = registerTag("reptrace") +var repDeepCopyTag = registerTag("repdeepcopy") +var repEnumToStrTag = registerTag("repenumtostr") +var repMethodTag = registerTag("repmethod") +#var repClassTag = registerTag("repclass") +var includeTag = registerTag("include") +var importTag = registerTag("import") +var implTag = registerTag("implementation") +var reexpModTag = registerTag("reexpmod") + +proc registerNifAstTags*() = + ## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag` + ## initializers above depend on `nifstreams.pool` having been initialized + ## FIRST (`pool = createLiterals(TagData)` in nifstreams' module init) — an + ## inter-module init-order requirement. The IC-built compiler currently emits + ## module init calls in a different order, so the initializers registered + ## into a pool that was subsequently replaced: the tag ids then denoted + ## builtin tags (`replay` came out as `deref`, `repdestroy` as `pat`, ...) + ## and every written NIF was silently corrupted. Called from `nim.nim` + ## before any command runs; idempotent (`getOrIncl` by name). + sdefTag = registerTag(symDefTagName) + tdefTag = registerTag(typeDefTagName) + hiddenTypeTag = registerTag(hiddenTypeTagName) + replayTag = registerTag("replay") + repConverterTag = registerTag("repconverter") + repDestroyTag = registerTag("repdestroy") + repWasMovedTag = registerTag("repwasmoved") + repCopyTag = registerTag("repcopy") + repSinkTag = registerTag("repsink") + repDupTag = registerTag("repdup") + repTraceTag = registerTag("reptrace") + repDeepCopyTag = registerTag("repdeepcopy") + repEnumToStrTag = registerTag("repenumtostr") + repMethodTag = registerTag("repmethod") + includeTag = registerTag("include") + importTag = registerTag("import") + implTag = registerTag("implementation") + reexpModTag = registerTag("reexpmod") proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n == nil: @@ -589,13 +689,28 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n[namePos].kind == nkSym: ast = n[namePos].sym.astImpl if ast == nil: ast = n - else: skipParams = true + else: + # params can only be recovered from `sym.typ.n` if the routine + # was actually semchecked. A routine nested in a TEMPLATE body + # (e.g. faststreams' `proc consumer(bytesVar: openArray[byte]) + # {.gensym.}` inside `consumeOutputs`) has a sym but a nil type — + # its params exist only in the AST; dropping them broke the + # template-param substitution at expansion ("undeclared + # identifier" for the injected name). + skipParams = n[namePos].sym.typImpl != nil w.withNode dest, ast: for i in 0 ..< ast.len: if i == paramsPos and skipParams: - # Parameter are redundant with s.typ.n and even dangerous as for generic instances - # we do not adapt the symbols properly - addDotToken(dest) + # Parameters are redundant with s.typ.n (and re-emitting their syms + # is dangerous for generic instances — we do not adapt the symbols + # properly). Emit an `nkEmpty` placeholder rather than a dot token: + # a dot loads back as a `nil` son, but ast children must be real + # nodes — the loaded routine ast is walked by passes (lambdalifting, + # liftdestructors, transf) that dereference `ast[paramsPos]`, and + # `nkEmpty` is the canonical empty slot. The actual params are + # recovered from `sym.typ.n` where needed. + dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)), NoLineInfo + dest.addParRi else: writeNode(w, dest, ast[i], forAst) dec w.inProc @@ -697,7 +812,10 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() of MethodEntry: - discard "to implement" + content.addParLe repMethodTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) + content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) + content.addParRi() of EnumToStrEntry: content.addParLe repEnumToStrTag, NoLineInfo content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) @@ -706,9 +824,385 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = of GenericInstEntry: discard "will only be written later to ensure it is materialized" +# --------------------------- Interface cookie --------------------------- +# +# Port of Nimony's `processForChecksum` (dist/nimony/src/lib/nifindexes.nim): +# ONE checksum per module over the importer-visible surface, stored in a tiny +# `.iface.nif` sidecar written OnlyIfChanged. deps.nim points the +# dependents' `nim_m` build edges at the sidecar instead of the bulky semmed +# NIF, so nifmake's mtime pruning stops the m-step cascade at the first +# module whose interface did not change. +# +# Hashed (importer-visible surface): +# - import/include/export entries, `(replay ...)` macro-cache actions and the +# rep* hook/converter/enumtostr registrations (all eagerly consumed by every +# importer's sem via processTopLevel/loadTransitiveHooks). +# - every EXPORTED `(sd ...)`: full content for consts/types/vars/lets; for +# EVERY routine kind (plain procs, templates, macros, iterators, generics, +# `inline` procs alike) only the SIGNATURE — the body is skipped. A routine +# body is invisible to a dependent's SEM unless the dependent expands / +# instantiates / VM-runs it, and each of those records a NeedsImpl (strong) +# edge gating the dependent on this module's IMPL cookie instead (see +# `cookieSd`). This keeps the iface cookie body-insensitive, so a body edit +# re-sems only the modules that actually consumed that body — not every +# importer (the old model folded inline-semantics bodies into the iface +# cookie, re-semming all importers on any such body edit). +# - nothing else: private defs and top-level init code are invisible to +# importers' sem (their effects on dependents' CODEGEN — and the codegen +# effect of inline iterator/proc body edits — are covered by the nifc +# backend's transitive NIF-mtime invalidation, which is unchanged). +# +# Token-content hashing only — line infos never enter the hash. Names DEFINED +# inside a hashed (sd) (params, locals, the embedded `(td `tK.item.mod)` defs) +# are replaced by per-sd ordinals and module-local `tK.item references are +# replaced by their structural td hash: both carry process-local mint counters +# that shift file-wide when an unrelated body creates a new type (measured: +# a single new instantiation renumbered every later signature), while +# dependents never reference them by name (verified over a full compiler +# cache: cross-module refs hit only top-level routine names). +# +# The cookie finally mixes in the DIRECT dependencies' sidecar contents +# ("hash chaining"): an interface change then propagates transitively +# level-by-level even when an intermediate module's own surface is unchanged +# (its sem still consumed the dep's surface, e.g. via the transitive hook +# replay). Chaining also guarantees a fired rule refreshes its sidecar mtime, +# which nifmake's max-output `needsRebuild` needs to not re-fire forever. +# +# The IMPL cookie (`.impl.nif`) complements it: a line-info-free hash +# of the module's ENTIRE content with the iface cookie mixed in. Dependents +# that consumed this module's bodies at compile time (recorded in the +# `.edges.nif` sidecar; see `ModuleGraph.icImplDeps`) are gated on it instead. + +type + CookieCtx = object + selfSuffix: string + tdRanges: Table[SymId, int] # td sym -> start of its first (td ...) tree + memo: Table[SymId, string] # td sym -> structural digest + expanding: HashSet[SymId] # cycle guard for recursive td expansion + depSuffixes: seq[string] # module suffixes of the direct imports + +proc nextTree(buf: TokenBuf; i: int): int = + ## Index just past the atom or balanced subtree starting at `i`. + result = i+1 + if buf[i].kind != ParLe: return + var nested = 0 + var j = i + while j < buf.len: + case buf[j].kind + of ParLe: inc nested + of ParRi: + dec nested + if nested == 0: return j+1 + else: discard + inc j + result = buf.len + +proc updateAtom(s: var Sha1State; t: PackedToken) = + # mirrors nimony's nifchecksums.update: token content only, no line infos + case t.kind + of ParLe: + s.update "(" + s.update pool.tags[t.tagId] + of ParRi: s.update ")" + of Ident: + s.update " " + s.update pool.strings[t.litId] + of StringLit: + s.update " \"" + s.update pool.strings[t.litId] + of IntLit: + s.update " " + s.update $pool.integers[t.intId] + of UIntLit: + s.update " " + s.update $pool.uintegers[t.uintId] + of FloatLit: + # hash the bit pattern, not a formatted float (no formatting variance) + s.update " f" + s.update $cast[uint64](pool.floats[t.floatId]) + of CharLit: + s.update " c" + s.update $t.uoperand + of DotToken: s.update "." + of UnknownToken: s.update "?" + of EofToken: s.update "!" + of Symbol, SymbolDef: discard "handled by hashRegion" + +proc isModuleLocalName(c: CookieCtx; name: string): bool = + let sn = parseSymName(name) + result = sn.module.len == 0 or sn.module == c.selfSuffix + +proc hashRegion(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; + start, theEnd: int; skipFrom = -1; skipTo = -1; + keepFirstDefLiteral = false) + +proc expandTd(c: var CookieCtx; buf: TokenBuf; name: SymId): string = + ## Structural digest of a module-local type def: hashes the `(td ...)` tree + ## instead of the volatile `tK.item counter name. Memoized; cycles fall back + ## to the literal name (sound — at worst a spurious cookie change). + if c.memo.hasKey(name): return c.memo[name] + if not c.tdRanges.hasKey(name) or c.expanding.contains(name): + return pool.syms[name] + c.expanding.incl name + let start = c.tdRanges[name] + var sub = newSha1State() + hashRegion(sub, c, buf, start, nextTree(buf, start)) + result = "&" & $SecureHash(sub.finalize()) + c.expanding.excl name + c.memo[name] = result + +proc hashRegion(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; + start, theEnd: int; skipFrom = -1; skipTo = -1; + keepFirstDefLiteral = false) = + # pass 1: assign ordinals to every symbol DEFINED in the hashed region + # (params, locals, embedded type defs). The region's own top-level name + # (first SymbolDef) stays literal when requested — it is what importers + # reference. + var ords = initTable[SymId, int]() + var first = keepFirstDefLiteral + var i = start + while i < theEnd: + if i == skipFrom: + i = skipTo + continue + if buf[i].kind == SymbolDef: + let sym = buf[i].symId + if first: + first = false + elif isModuleLocalName(c, pool.syms[sym]) and not ords.hasKey(sym): + ords[sym] = ords.len + inc i + # pass 2: hash + first = keepFirstDefLiteral + i = start + while i < theEnd: + if i == skipFrom: + i = skipTo + continue + let t = buf[i] + if t.kind in {Symbol, SymbolDef}: + let sym = t.symId + let name = pool.syms[sym] + s.update(if t.kind == SymbolDef: " :" else: " ") + if t.kind == SymbolDef and first: + first = false + s.update name + elif ords.hasKey(sym): + s.update "%" + s.update $ords[sym] + elif name.startsWith("`t") and isModuleLocalName(c, name): + s.update expandTd(c, buf, sym) + else: + s.update name + else: + updateAtom s, t + inc i + +proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): int = + ## Contributes one `(sd ...)` subtree to the cookie; returns the index past it. + result = nextTree(buf, start) + if buf[start+1].kind != SymbolDef: return + let marker = buf[start+2] + if not (marker.kind == Ident and pool.strings[marker.litId] == "x"): + return # not importable -> invisible to dependents' sem (nimony parity) + # field layout, see writeSymDef: kind magic flags options offset position + # annex type owner ast loc constraint instantiatedFrom + var fields: array[13, int] = default(array[13, int]) + var i = start + 3 + for f in 0 ..< 13: + fields[f] = i + i = nextTree(buf, i) + var kind = skUnknown + {.cast(uncheckedAssign).}: + kind = parse(TSymKind, pool.tags[buf[fields[0]].tagId]) + var skipFrom = -1 + var skipTo = -1 + if kind in routineKinds: + # Routines contribute their SIGNATURE only to the iface cookie. A routine + # body is invisible to a dependent's SEM unless the dependent expands, + # instantiates, or VM-runs it — and each of those records a NeedsImpl + # (strong) edge that gates the dependent on this module's IMPL cookie + # instead (templates -> semTemplateExpr, generics -> generateInstance, + # macros/compile-time procs -> the VM's genProc, getImpl -> opcGetImpl). + # Inline iterators and `inline`-callconv procs are inlined at codegen; the + # nifc backend's transitive NIF-mtime invalidation re-codegens their users. + # So no routine body needs to live in the iface cookie. + let ast = fields[9] + if buf[ast].kind == ParLe: + # skip son `bodyPos` (6) of the routine ast tree; NOT the last element — + # sem appends the result sym at `resultPos` (7) after the body. + let astEnd = nextTree(buf, ast) + var p = ast + 1 # the flags atom + var ok = true + for _ in 0 ..< 2 + bodyPos: # flags, type, sons 0..5 + p = nextTree(buf, p) + if p >= astEnd - 1: + ok = false + break + if ok: + skipFrom = p + skipTo = nextTree(buf, p) + # non-routine kinds (consts carry their value, types their structure incl. + # default field values): hash everything. + hashRegion(s, c, buf, start, result, skipFrom, skipTo, keepFirstDefLiteral = true) + +proc scanStmtsForCookie(s: var Sha1State; c: var CookieCtx; buf: TokenBuf) = + ## Walks the whole written module, hashing only the importer-visible pieces; + ## unknown structure is descended into (var/let/type section wrappers, + ## top-level code) but contributes nothing itself — nimony-style. + let exportTag = pool.tags.getOrIncl(toNifTag(nkExportStmt)) + let exportExceptTag = pool.tags.getOrIncl(toNifTag(nkExportExceptStmt)) + var i = 0 + while i < buf.len: + let t = buf[i] + if t.kind == ParLe: + let tid = t.tagId + if tid == sdefTag: + i = cookieSd(s, c, buf, i) + elif tid == implTag: + i = nextTree(buf, i) + elif tid == replayTag or tid == repConverterTag or tid == repDestroyTag or + tid == repWasMovedTag or tid == repCopyTag or tid == repSinkTag or + tid == repDupTag or tid == repTraceTag or tid == repDeepCopyTag or + tid == repEnumToStrTag or tid == repMethodTag or + tid == exportTag or tid == exportExceptTag or tid == includeTag: + let e = nextTree(buf, i) + hashRegion(s, c, buf, i, e) + i = e + elif tid == importTag: + let e = nextTree(buf, i) + hashRegion(s, c, buf, i, e) + for j in i ..< e: + if buf[j].kind == StringLit: + let suffix = pool.strings[buf[j].litId] + if suffix notin c.depSuffixes: c.depSuffixes.add suffix + i = e + else: + inc i # descend without hashing + else: + inc i + +proc icGroupSuffixes(config: ConfigRef): HashSet[string] = + ## Module suffixes of the --icGroup cycle members compiled by this very + ## process (their sidecars are being produced concurrently, so neither + ## chaining nor edge recording may depend on them). + result = initHashSet[string]() + for p in config.icGroup: + result.incl cachedModuleSuffix(config, fileInfoIdx(config, AbsoluteFile p)) + +proc writeCookieFile(config: ConfigRef; selfSuffix, tag, hex, ext: string) = + var dest = createTokenBuf(4) + dest.addParLe pool.tags.getOrIncl(tag), NoLineInfo + dest.addStrLit hex + dest.addParRi + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ext).string + writeFile(dest, path, OnlyIfChanged) + +proc writeIfaceCookie(config: ConfigRef; thisModule: int32; buf: TokenBuf): string = + let selfSuffix = modname(thisModule, config) + var c = CookieCtx(selfSuffix: selfSuffix) + # pre-pass: first (td ...) occurrence per type name, wherever it is embedded + var i = 0 + while i < buf.len: + if buf[i].kind == ParLe and buf[i].tagId == tdefTag and i+1 < buf.len and + buf[i+1].kind == SymbolDef: + let nm = buf[i+1].symId + if not c.tdRanges.hasKey(nm): c.tdRanges[nm] = i + inc i + var s = newSha1State() + scanStmtsForCookie(s, c, buf) + # chain the direct deps' cookies; co-members of an --icGroup cycle are + # excluded (their sidecars are being produced by this very rule — chaining + # them would make the hash depend on within-group write order). + let groupSuffixes = icGroupSuffixes(config) + for dep in c.depSuffixes: + if dep == selfSuffix or dep in groupSuffixes: continue + let depIface = toGeneratedFile(config, AbsoluteFile(dep), ".iface.nif").string + s.update "|" + s.update dep + s.update ":" + s.update(try: readFile(depIface) except IOError, OSError: "") + result = $SecureHash(s.finalize()) + writeCookieFile(config, selfSuffix, "iface", result, ".iface.nif") + +proc writeImplCookie(config: ConfigRef; thisModule: int32; buf: TokenBuf; + ifaceHex: string) = + ## The implementation cookie: a line-info-free hash of the module's ENTIRE + ## serialized content (private defs and routine bodies included), with the + ## module's own iface cookie mixed in so impl sensitivity is a strict + ## superset of iface sensitivity (incl. the chained dep ifaces — a NeedsImpl + ## edge REPLACES the iface edge, it must not lose its triggers). Dependents + ## that consumed this module's bodies at compile time are gated on this file + ## instead of the iface cookie. Comment-only edits move neither cookie. + ## No id normalization here: a counter shift implies some real content + ## change elsewhere in the module, which flips the hash anyway — and any + ## body change is exactly what NeedsImpl dependents must see. + let selfSuffix = modname(thisModule, config) + var s = newSha1State() + for i in 0 ..< buf.len: + let t = buf[i] + if t.kind in {Symbol, SymbolDef}: + s.update(if t.kind == SymbolDef: " :" else: " ") + s.update pool.syms[t.symId] + else: + updateAtom s, t + s.update "|iface:" + s.update ifaceHex + writeCookieFile(config, selfSuffix, "impl", $SecureHash(s.finalize()), ".impl.nif") + +proc writeEdgesFile(config: ConfigRef; thisModule: int32; implDeps: seq[int]) = + ## Records which modules' bodies this compilation consumed at compile time + ## (`ModuleGraph.icImplDeps`): the NeedsImpl edge set. deps.nim reads this + ## sidecar when regenerating the build file and gates this module on those + ## dependencies' IMPL cookies instead of their iface cookies. + let selfSuffix = modname(thisModule, config) + let groupSuffixes = icGroupSuffixes(config) + var suffixes: seq[string] = @[] + for id in implDeps: + if id == thisModule.int: continue + let suffix = cachedModuleSuffix(config, FileIndex id) + if suffix.len == 0 or suffix == selfSuffix or suffix in groupSuffixes: + continue + if suffix notin suffixes: suffixes.add suffix + sort suffixes + var dest = createTokenBuf(4 + 2*suffixes.len) + dest.addParLe pool.tags.getOrIncl("edges"), NoLineInfo + for suffix in suffixes: + dest.addStrLit suffix + dest.addParRi + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".edges.nif").string + # Deliberately ALWAYS written (unlike every other output of the nim_m rule): + # nothing gates on this file's mtime — deps.nim only reads its content — so + # it doubles as the rule's freshness stamp. nifmake's `needsRebuild` takes + # the freshest output as proof of "ran since the inputs changed"; without an + # always-written output a rule whose re-run produces only content-identical + # (mtime-preserved) files would re-fire on every warm build (e.g. after an + # edit was reverted). Nimony's analog is its always-written `.s.nif`. + writeFile(dest, path) + +proc writeSemDeps*(config: ConfigRef; thisModule: int32; importPaths: seq[string]) = + ## The module's REAL direct imports as `nim m` sem resolved them — static + ## plus any a macro generated — recorded as full source paths. `nim ic` reads + ## this `.s.deps.nif` to re-derive the build graph: imports the static scanner + ## missed become new nodes (replacing the old build-failure discovery loop), + ## and `when false` imports the scanner over-included are pruned. Always + ## written so it is current after every successful sem (like `.edges`). + let selfSuffix = modname(thisModule, config) + var paths = importPaths + sort paths + var dest = createTokenBuf(4 + 2*paths.len) + dest.addParLe pool.tags.getOrIncl("semdeps"), NoLineInfo + for p in paths: + dest.addStrLit p + dest.addParRi + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".s.deps.nif").string + writeFile(dest, path) + proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; opsLog: seq[LogEntry]; - replayActions: seq[PNode] = @[]) = + replayActions: seq[PNode] = @[]; + implDeps: seq[int] = @[]; + reexportedModules: seq[(string, string)] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var content = createTokenBuf(300) @@ -729,6 +1223,17 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; var bottom = createTokenBuf(300) w.writeToplevelNode content, bottom, n + # Re-exported MODULES (`import x; export x`): semExport puts only x's + # member syms into the nkExportStmt; the module sym itself reaches the + # exporter's interface via `reexportSym` and acts as a QUALIFIER there + # (`asmm.x86.nd`). Serialize (name, suffix) pairs so the loader can + # rebuild that part of the interface. + for (mname, msuffix) in reexportedModules: + w.deps.addParLe reexpModTag, NoLineInfo + w.deps.addStrLit mname + w.deps.addStrLit msuffix + w.deps.addParRi + # the implTag is used to tell the loader that the # bottom of the file is the implementation of the module: content.addParLe implTag, NoLineInfo @@ -758,7 +1263,15 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; dest.addParRi() - writeFile(dest, d) + # OnlyIfChanged keeps the mtime of content-identical rewrites: nifmake's + # mtime-based `needsRebuild` then prunes the rebuild cascade level by + # level, and the nifc backend can trust "semmed NIF older than the cnif + # artifact" as an honest per-module unchanged stamp. + writeFile(dest, d, OnlyIfChanged) + if not isDefined(config, "icNoIfaceGate"): + let ifaceHex = writeIfaceCookie(config, thisModule, dest) + writeImplCookie(config, thisModule, dest, ifaceHex) + writeEdgesFile(config, thisModule, implDeps) # --------------------------- Loader (lazy!) ----------------------------------------------- @@ -819,6 +1332,8 @@ type symCounter: int32 index: Table[string, NifIndexEntry] # Simple embedded index for offsets suffix: string + contentStart: int # stream offset of the module body, so a full-AST load can + # rewind after lazy symbol loads moved the cursor DecodeContext* = object infos: LineInfoWriter @@ -827,11 +1342,32 @@ type syms: Table[string, (PSym, NifIndexEntry)] mods: Table[FileIndex, NifModule] cache: IdentCache + mainModuleSuffix: string + ## Mangled module name of the module being compiled fresh (cmdM). Symbols + ## belonging to it that are re-exported by a dependency must NOT be loaded + ## as stubs, otherwise they collide with the freshly compiled originals. proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache) +proc setMainModule*(c: var DecodeContext; fileIdx: FileIndex) = + ## Records the module that is being compiled fresh so that re-exports of its + ## own symbols by dependencies are not turned into duplicate stubs. + c.mainModuleSuffix = modname(fileIdx.int, c.infos.config) + +proc getMainModuleSuffix*(c: DecodeContext): string {.inline.} = + c.mainModuleSuffix + +proc loadedState(c: DecodeContext): ItemState {.inline.} = + ## State to give a freshly loaded symbol or type. During the C code generation + ## phase (`nim nifc`) the backend (lambda lifting, the transformer, etc.) + ## legitimately mutates the loaded entities and never writes them back to a NIF, + ## so they must be mutable (`Complete`). During semantic checking (`nim m`) a + ## loaded entity belongs to an already-compiled dependency and must stay + ## `Sealed` so accidental mutations are caught. + if c.infos.config.cmd == cmdNifC: Complete else: Sealed + proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry; buf: var TokenBuf): Cursor = let s = addr c.mods[module].stream @@ -895,7 +1431,10 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): "whose NIF file hasn't been written yet." var stream = nifstreams.open(modFile) let index = readEmbeddedIndex(stream) - c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix) + # `readEmbeddedIndex` leaves the cursor at the start of the module body. + let contentStart = offset(stream.r) + c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix, + contentStart: contentStart) proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry = let ii = addr c.mods[module].index @@ -920,14 +1459,17 @@ proc createTypeStub(c: var DecodeContext; t: SymId): PType = k = k * 10 + name[i].ord - ord('0') inc i if i < name.len and name[i] == '.': inc i - var itemId = 0'i32 + var itemVal = 0'i32 while i < name.len and name[i] in {'0'..'9'}: - itemId = itemId * 10'i32 + int32(name[i].ord - ord('0')) + itemVal = itemVal * 10'i32 + int32(name[i].ord - ord('0')) inc i if i < name.len and name[i] == '.': inc i let suffix = name.substr(i) - let id = ItemId(module: moduleId(c, suffix).int32, item: itemId) - let offs = c.getOffset(id.module.FileIndex, name) + let id = itemId(moduleId(c, suffix).int32, itemVal) + let ii = addr c.mods[id.module.FileIndex].index + let offs = ii[].getOrDefault(name) + if offs.offset == 0: + raiseAssert "symbol has no offset: " & name result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) c.types[name] = (result, offs) @@ -955,7 +1497,7 @@ proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: s let module = moduleId(c, thisModule) let val = addr c.mods[module].symCounter inc val[] - let id = ItemId(module: module.int32, item: val[]) + let id = itemId(module.int32, val[]) let sym = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Complete) localSyms[symName] = sym @@ -963,7 +1505,7 @@ proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: s # We're currently at the `(sd` position, need to skip to SymbolDef inc n # skip past `sd` tag to get to SymbolDef loadSymFromCursor(c, sym, n, thisModule, localSyms) - sym.state = Sealed # mark as fully loaded + sym.state = c.loadedState # mark as fully loaded # Continue processing - loadSymFromCursor already advanced n past the closing `)` continue inc depth @@ -988,7 +1530,7 @@ proc loadTypeStub(c: var DecodeContext; n: var Cursor; localSyms: var Table[stri let s = n.firstSon.symId result = createTypeStub(c, s) if result.state == Partial: - result.state = Sealed # Mark as loaded to prevent loadType from re-loading with empty localSyms + result.state = c.loadedState # Mark as loaded to prevent loadType from re-loading with empty localSyms loadTypeFromCursor(c, n, result, localSyms) else: skip n # Type already loaded, skip over the td block @@ -1014,10 +1556,11 @@ proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string; let module = moduleId(c, sn.module) let val = addr c.mods[module].symCounter inc val[] - let id = ItemId(module: module.int32, item: val[]) + let id = itemId(module.int32, val[]) let offs = c.getOffset(module, symAsStr) - result = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) + let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) + result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, offs) proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string; @@ -1034,7 +1577,8 @@ proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string; skip n result = loadSymStub(c, s, thisModule, localSyms) else: - raiseAssert "sym expected but got " & $n.kind + raiseAssert "sym expected but got " & $n.kind & ( + if n.kind == Ident: " '" & pool.strings[n.litId] & "'" else: "") proc isStub*(t: PType): bool {.inline.} = t.state == Partial proc isStub*(s: PSym): bool {.inline.} = s.state == Partial @@ -1097,7 +1641,14 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms loadField t.sizeImpl loadField t.alignImpl loadField t.paddingAtEndImpl - loadField t.itemId.item # nonUniqueId + t.itemId = itemId(t.itemId.module, loadAtom(int32, n)) # nonUniqueId + if n.kind == StringLit: + # itemId.module differs from uniqueId.module (an `exactReplica` of a + # foreign type): restore the canonical module half + t.itemId = itemId(int32(moduleId(c, pool.strings[n.litId])), t.itemId.item) + inc n + elif n.kind == DotToken: + inc n t.typeInstImpl = loadTypeStub(c, n, localSyms) t.nImpl = loadNode(c, n, typesModule, localSyms) @@ -1112,7 +1663,7 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms proc loadType*(c: var DecodeContext; t: PType) = if t.state != Partial: return - t.state = Sealed + t.state = c.loadedState var buf = createTokenBuf(30) let typeName = typeToNifSym(t, c.infos.config) var n = cursorFromIndexEntry(c, t.itemId.module.FileIndex, c.types[typeName][1], buf) @@ -1159,6 +1710,11 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: s.kindImpl = parse(TSymKind, pool.tags[n.tagId]) inc n + if s.kindImpl == skPackage and s.name.s.endsWith(PkgMarker): + # Fallback: stubs are normally created with the clean name already + # (see stubKindAndName); strip the NIF-only marker if one slipped through. + s.name = c.cache.getIdent(s.name.s[0 ..< s.name.s.len - PkgMarker.len]) + case s.kindImpl of skLet, skVar, skField, skForVar: s.guardImpl = loadSymStub(c, n, thisModule, localSyms) @@ -1199,7 +1755,7 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: proc loadSym*(c: var DecodeContext; s: PSym) = if s.state != Partial: return - s.state = Sealed + s.state = c.loadedState var buf = createTokenBuf(30) let symsModule = s.itemId.module.FileIndex let nifname = globalName(s, c.infos.config) @@ -1285,14 +1841,14 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; let module = moduleId(c, thisModule) let val = addr c.mods[module].symCounter inc val[] - let id = ItemId(module: module.int32, item: val[]) + let id = itemId(module.int32, val[]) sym = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Complete) localSyms[symName] = sym # register for later references # Now fully load the symbol from the sdef inc n # skip `sd` tag loadSymFromCursor(c, sym, n, thisModule, localSyms) - sym.state = Sealed # mark as fully loaded + sym.state = c.loadedState # mark as fully loaded result = newSymNode(sym, info) else: sym = c.loadSymStub(name.symId, thisModule, localSyms) @@ -1391,8 +1947,9 @@ proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; let val = addr c.mods[symModule].symCounter inc val[] - let id = ItemId(module: symModule.int32, item: val[]) - result = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) + let id = itemId(symModule.int32, val[]) + let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) + result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, entry) proc extractBasename(nifName: string): string = @@ -1428,6 +1985,29 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; # Move index table back c.mods[module].index = move indexTab +proc moduleSymbolStubs*(c: var DecodeContext; module: FileIndex): seq[PSym] = + ## Stubs for every non-type symbol serialized in `module`'s NIF index. The + ## per-module backend uses this to emit the routines a module OWNS: procs are + ## serialized as `(sd ...)` symbol-defs and loaded lazily, never as + ## `nkProcDef` statements in the top-level stmt list, so `genTopLevelStmt` + ## alone never reaches them — without this, a routine called only from other + ## modules would be emitted by nobody once the demanding module merely + ## prototypes it. + ## + ## Returns lazy stubs: the index table is moved out while iterating (loading a + ## symbol can register new modules and invalidate the iterator), so the caller + ## forces full load (`.kind`, `.ast`) and filters AFTER this returns, with the + ## index back in place. + result = @[] + if not c.mods.hasKey(module): return + var indexTab = move c.mods[module].index + let thisModule = c.mods[module].suffix + for nifName, entry in indexTab: + if nifName.startsWith("`t"): continue # types are not routines + let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) + if sym != nil: result.add sym + c.mods[module].index = move indexTab + proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string @@ -1456,7 +2036,7 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo # Create a stub symbol let val = addr c.mods[module].symCounter inc val[] - let id = ItemId(module: int32(module), item: val[]) + let id = itemId(int32(module), val[]) result = PSym(itemId: id, kindImpl: skProc, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, offs) @@ -1469,10 +2049,29 @@ proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym = proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: FileIndex): PSym = ## Tries to resolve a compiler proc from a module by checking the NIF index. - ## Returns nil if the symbol doesn't exist. + ## Returns nil if the symbol doesn't exist. The NIF disamb is mint order, so + ## `name.0.` can be any of the overloads sharing the name — for `newSeq` it + ## is the generic magic, not the RTL proc (a refc build then demands codegen + ## of the generic and dies on `seq[T]`): enumerate the index entries with + ## this basename and pick the one that carries `sfCompilerProc`. + result = nil let suffix = moduleSuffix(c.infos.config, moduleFileIdx) - let symName = name & ".0." & suffix - result = resolveSym(c, symName, true) + let module = moduleId(c, suffix) + let prefix = name & "." + var candidates: seq[int] = @[] + for key in c.mods[module].index.keys: + if key.len > prefix.len and key.startsWith(prefix): + let sn = parseSymName(key) + if sn.name == name: + candidates.add sn.count + # the loads below can grow `c.mods` (symbols reference other modules), so + # resolve only after the index iteration is done + for count in candidates: + let sym = resolveSym(c, name & "." & $count & "." & suffix, true) + if sym != nil: + loadSym(c, sym) + if sfCompilerProc in sym.flagsImpl: + return sym proc loadLogOp(c: var DecodeContext; logOps: var seq[LogEntry]; s: var Stream; kind: LogEntryKind; op: TTypeAttachedOp; module: int): PackedToken = result = next(s) @@ -1527,6 +2126,8 @@ type deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of logOps*: seq[LogEntry] module*: PSym # set by modulegraphs.nim! + reexportedModules*: seq[(string, string)] # (name, suffix) of re-exported MODULE syms; + # materialized by modulegraphs.nim proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) = tok = next(s) # skip `(import` @@ -1544,6 +2145,25 @@ proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix] else: raiseAssert "expected ParRi but got " & $tok.kind +proc addReexportedEnumFields(c: var DecodeContext; sym: PSym; interf: var TStrTable) = + ## When a non-pure enum type is (re-)exported, its fields must also become + ## visible (unqualified) to importers. In a from-source build this happens via + ## `rawImportSymbol`'s enum handling when the type is imported; the lazy IC + ## importer never runs that, so we materialise the fields into the interface + ## here, when the export list is processed. + loadSym(c, sym) + if sym.kindImpl != skType or sfPure in sym.flagsImpl: return + let et = sym.typImpl + if et == nil: return + loadType(c, et) + if et.kind notin {tyEnum, tyBool}: return + let fields = et.nImpl + if fields == nil: return + for i in 0 ..< fields.len: + let f = fields[i] + if f != nil and f.kind == nkSym and f.sym != nil: + strTableAdd(interf, f.sym) + proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; interf: var TStrTable; suffix: string; module: int): PrecompiledModule = result = PrecompiledModule(topLevel: newNode(nkStmtList)) @@ -1593,6 +2213,7 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; #elif t.tagId == repClassTag: # t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module) elif t.tagId == exportTag: + var lastGood = "" t = next(s) # skip (export if t.kind == DotToken: t = next(s) # skip dot @@ -1601,19 +2222,53 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; while true: if t.kind == Symbol: let symAsStr = pool.syms[t.symId] - let sym = resolveSym(c, symAsStr, false) - if sym != nil: - strTableAdd(interf, sym) + lastGood = symAsStr + # Skip symbols that are re-exported by this dependency but actually + # belong to the module we are compiling fresh: loading them as stubs + # would shadow/collide with the freshly compiled originals. + if c.mainModuleSuffix.len == 0 or + parseSymName(symAsStr).module != c.mainModuleSuffix: + # Resolving an exported symbol of this very module (`export` of a + # symbol that lives in a `when` branch of the same file) lazily + # loads it from the stream we are currently iterating, moving the + # cursor into the symbol's `(sd ...)` definition. Save/restore the + # position so the export-list parse continues where it left off. + let saved = offset(s.r) + let sym = resolveSym(c, symAsStr, false) + if sym != nil: + strTableAdd(interf, sym) + addReexportedEnumFields(c, sym, interf) + s.r.jumpTo(saved) t = next(s) elif t.kind == ParRi: break else: - raiseAssert "expected Symbol or ParRi but got " & $t.kind + raiseAssert "expected Symbol or ParRi but got " & $t.kind & + " (" & (if t.kind == ParLe: pool.tags[t.tagId] else: "") & + ") in export list of module " & suffix & ", last symbol: " & lastGood t = next(s) elif t.tagId == includeTag: t = skipTree(s) elif t.tagId == importTag: loadImport(c, s, result.deps, t) + elif t.tagId == reexpModTag: + # a re-exported MODULE: (reexpmod "name" "suffix"); the module sym + # is a qualifier in this module's interface — materialized by the + # caller (modulegraphs), which can register interface tables + t = next(s) + var mname = "" + var msuffix = "" + if t.kind == StringLit: + mname = pool.strings[t.litId] + t = next(s) + if t.kind == StringLit: + msuffix = pool.strings[t.litId] + t = next(s) + if t.kind != ParRi: + raiseAssert "expected ParRi in reexpmod entry of module " & suffix + t = next(s) + if mname.len > 0 and msuffix.len > 0: + result.reexportedModules.add (mname, msuffix) elif t.tagId == implTag: cont = false elif LoadFullAst in flags: @@ -1636,8 +2291,11 @@ proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHi let module = moduleId(c, string(suffix), flags) # Load the module AST (or just replay actions if loadFullAst is false) - # processTopLevel also collects export instructions + # processTopLevel also collects export instructions. + # Lazy symbol loading may have moved the stream cursor since the module was + # opened, so rewind to the start of the module body before reading it. let s = addr c.mods[module].stream + s[].r.jumpTo(c.mods[module].contentStart) var t = next(s[]) if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): t = next(s[]) # skip (stmts @@ -1662,3 +2320,4 @@ when isMainModule: echo obj.name, " ", obj.module, " ", obj.count let objb = parseSymName("abcdef.0121") echo objb.name, " ", objb.module, " ", objb.count + diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 6a7b4d7788..5857531c8d 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -20,6 +20,9 @@ export int128 import nodekinds export nodekinds +import itemids +export itemids + type TCallingConvention* = enum ccNimCall = "nimcall" # nimcall, also the default @@ -571,23 +574,6 @@ const generatedMagics* = {mNone, mIsolate, mFinished, mOpenArrayToSeq} ## magics that are generated as normal procs in the backend -type - ItemId* = object - module*: int32 - item*: int32 - -proc `$`*(x: ItemId): string = - "(module: " & $x.module & ", item: " & $x.item & ")" - -proc `==`*(a, b: ItemId): bool {.inline.} = - a.item == b.item and a.module == b.module - -proc hash*(x: ItemId): Hash = - var h: Hash = hash(x.module) - h = h !& hash(x.item) - result = !$h - - type PNode* = ref TNode TNodeSeq* = seq[PNode] diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index b2521069d4..feb5babeac 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -394,7 +394,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; n # variable. Thus, we create a temporary pointer variable instead. let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray if needsIndirect: - n.typ = n.typ.exactReplica + n.typ = n.typ.exactReplica(p.module.idgen) n.typ.incl tfVarIsPtr a = initLocExprSingleUse(p, n) a = withTmpIfNeeded(p, a, needsTmp) @@ -909,6 +909,16 @@ proc isInactiveDestructorCall(p: BProc, e: PNode): bool = proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = if p.withinBlockLeaveActions > 0 and isInactiveDestructorCall(p, ri): return + when defined(icDbgHash): + if ri[0].typ == nil: + echo "NILCALLEE kind=", ri[0].kind, + " sym=", (if ri[0].kind == nkSym: ri[0].sym.name.s else: "-"), + " symKind=", (if ri[0].kind == nkSym: $ri[0].sym.kind else: "-"), + " flags=", (if ri[0].kind == nkSym: $ri[0].sym.flags else: "-"), + " lazy=", nfLazyType in ri[0].flags, + " inProc=", (if p.prc != nil: p.prc.name.s else: "NIL"), + " module=", p.module.module.name.s + raiseAssert "nil callee type, see NILCALLEE above" if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure: genClosureCall(p, le, ri, d) elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags: diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index c6e6057223..2c1bf17590 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3491,7 +3491,23 @@ proc genConstDefinition(q: BModule; p: BProc; sym: PSym) = data.addDeclWithVisibility(Private): data.addVarWithInitializer(Local, actualConstName, typ = td): genBracedInit(q.initProc, sym.astdef, isConst = true, sym.typ, data) - q.s[cfsData].add(extract(data)) + if q.config.cmd == cmdNifC: + # Each `cg` process that demands this const emits its definition + # (emit-everywhere). Always declare it first (the data analogue of a proc + # prototype) so a TU whose copy the merge stage drops still has a valid + # declaration; wrap the definition as a droppable `'d'` unit the merge + # stage assigns to a single owner. + let cname = stripCnifMarks(actualConstName) + var decl = newBuilder("") + decl.addDeclWithVisibility(Extern): + decl.addVar(kind = Local, name = actualConstName, typ = td) + q.s[cfsData].add(extract(decl)) + q.s[cfsData].add(cnifDefDirective(cname, "d", icNifName(q, sym))) + q.s[cfsData].add(extract(data)) + q.s[cfsData].add(cnifEndDefs()) + q.icDataDefs.add (cname, icNifName(q, sym)) + else: + q.s[cfsData].add(extract(data)) if q.hcrOn: # generate the global pointer with the real name q.s[cfsVars].addVar(kind = Global, name = sym.loc.snippet, @@ -3555,6 +3571,17 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of skProc, skConverter, skIterator, skFunc: #if sym.kind == skIterator: # echo renderTree(sym.getBody, {renderIds}) + if p.config.cmd == cmdNifC and + (isGenericRoutineStrict(sym) or sfCompileTime in sym.flags or + (sym.kind == skIterator and sym.typ.callConv == ccInline)): + # Under IC a module's top-level routine definitions are serialized as bare + # symbol references that reappear in the loaded statement list. Uninstantiated + # generic routines (incl. those with type-class params like `tuple`) and + # `.compileTime` routines have no run-time code, so skip them here. + # Inline iterators likewise have no standalone code — they are always inlined + # at their for-loop call sites by the transformer (only closure iterators get + # a standalone C function), so a bare serialized def reference is a no-op. + return if sfCompileTime in sym.flags: localError(p.config, n.info, "request to generate code for .compileTime proc: " & sym.name.s) @@ -3629,6 +3656,11 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = # echo renderTree(p.prc.ast, {renderIds}) internalError(p.config, n.info, "expr: param not init " & sym.name.s & "_" & $sym.id) putLocIntoDest(p, d, sym.loc) + of skTemplate, skMacro: + # Under IC a module's top-level template/macro definitions are serialized as + # bare symbol references (only their interface matters), so they reappear in + # the loaded statement list. They are compile-time only and produce no code. + discard else: internalError(p.config, n.info, "expr(" & $sym.kind & "); unknown symbol") of nkNilLit: if not isEmptyType(n.typ): diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index bc9c06fa1d..a30c1b1bf2 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1986,4 +1986,9 @@ proc genStmts(p: BProc, t: PNode) = if isPush: pushInfoContext(p.config, t.info) expr(p, t, a) if isPush: popInfoContext(p.config) - internalAssert p.config, a.k in {locNone, locTemp, locLocalVar, locExpr} + # A bare `nkSym` statement is how IC serializes a definition that lives inside a + # top-level block (e.g. a nested `proc`/`var`): codegen emits the definition and + # leaves the symbol's own location in `a` (e.g. `locProc`), which is discarded + # here, so the value-sanity check below does not apply to it. + internalAssert p.config, t.kind == nkSym or + a.k in {locNone, locTemp, locLocalVar, locExpr} diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 8e6b44c81d..eb82c854c7 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -72,6 +72,37 @@ proc mangleProc(m: BModule; s: PSym; makeUnique: bool): string = else: m.g.mangledPrcs.incl(result) +proc sharedInstanceCName(m: BModule; s: PSym): string = + ## The module-free canonical C name for a content-keyed generic instance, + ## or "" when the symbol must keep its module-suffixed name. With a shared + ## name, every TU that instantiated the same generic with the same type + ## arguments calls one extern definition (first claimant's TU embeds it, + ## see `genProcLvl3`) instead of compiling its own static copy. + ## + ## The name is program-unique only if the 30-bit content hash does not + ## collide for same-named instances of *different* instantiations across + ## modules — the per-module probe in `setInstanceDisamb` cannot see that. + ## Claimants therefore must present the same signature; on mismatch the + ## later one keeps its module-suffixed name (no merge, still correct). + ## Residual risk: same name and signature, different generic args, AND a + ## 30-bit collision — vanishingly unlikely; a full-typeKey verification + ## channel can close it later. + result = "" + if m.config.cmd == cmdNifC and s.kind in routineKinds and + (s.disamb and InstanceDisambBit) != 0'i32 and + s.typ != nil and s.typ.callConv != ccInline and not m.hcrOn and + {sfImportc, sfExportc, sfCodegenDecl} * s.flags == {}: + # The content-derived `disamb` is unique per process (collision-probed in + # `setInstanceDisamb`), so the mint-site-independent `_i` name is + # safe to use directly; identical instances across modules collide on it + # exactly and the merge stage keeps one. + result = s.name.s.mangle & "_i" & $s.disamb + +proc isSharedInstanceCName(m: BModule; s: PSym): bool = + m.config.cmd == cmdNifC and s.kind in routineKinds and + (s.disamb and InstanceDisambBit) != 0'i32 and + stripCnifMarks(s.loc.snippet) == s.name.s.mangle & "_i" & $s.disamb + proc fillBackendName(m: BModule; s: PSym) = if s.loc.snippet == "": var result: Rope @@ -79,13 +110,22 @@ proc fillBackendName(m: BModule; s: PSym) = m.g.config.symbolFiles == disabledSf: result = mangleProc(m, s, false).rope else: - result = s.name.s.mangle.rope - result.add mangleProcNameExt(m.g.graph, s) + let shared = sharedInstanceCName(m, s) + if shared.len > 0: + result = shared.rope + else: + result = s.name.s.mangle.rope + result.add mangleProcNameExt(m.g.graph, s) if m.hcrOn: result.add '_' result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config)) backendEnsureMutable s - s.locImpl.snippet = result + if m.config.cmd == cmdNifC: + # mark the name so the cnif artifact writer can turn every occurrence + # into a Symbol token; stripped from the actual C output in genModule + s.locImpl.snippet = markCName(result) + else: + s.locImpl.snippet = result proc fillParamName(m: BModule; s: PSym) = if s.loc.snippet == "": @@ -373,6 +413,12 @@ proc getSimpleTypeDesc(m: BModule; typ: PType): Rope = m.typeCache[sig] = result proc pushType(m: BModule; typ: PType) = + when defined(icDbgRefc): + if typ.kind == tySequence and + typ.elementType.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyGenericParam: + echo "[icRefc] pushType seq-of-genericparam t=", typeToString(typ), + " itemId=", typ.itemId.module, ".", typ.itemId.item, " mod=", m.module.name.s + echo getStackTrace() for i in 0..high(m.typeStack): # pointer equality is good enough here: if m.typeStack[i] == typ: return @@ -618,6 +664,18 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder, for i in 1..