diff --git a/.gitignore b/.gitignore index efb7dfe61e..98a3c18fcf 100644 --- a/.gitignore +++ b/.gitignore @@ -87,6 +87,7 @@ tweeter_test.db /tests/megatest.nim /tests/ic/*_temp.nim +/tests/ic/*_mm/ /tests/navigator/*_temp.nim diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 0504d18aca..839997e50c 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -11,8 +11,10 @@ import std / [assertions, tables, sets] from std / strutils import startsWith, endsWith, contains -from std / os import fileExists, dirExists, walkFiles -from std / syncio import readFile +from std / os import fileExists, dirExists, walkFiles, existsEnv, + commandLineParams, getCurrentProcessId +from std / exitprocs import addExitProc +from std / syncio import readFile, stderr, writeLine from std / algorithm import sort import "../dist/checksums/src/checksums" / sha1 import astdef, idents, msgs, options @@ -27,7 +29,7 @@ import "../dist/nimony/src/lib" / [bitabs, nifstreams, lineinfos, # uses; the reader reaches pools via `symName(c)`/`strVal(c)` etc. import "../dist/nimony/src/lib/nifcore" except pool from "../dist/nimony/src/lib" / bif import load, BifModule -import "../dist/nimony/src/gear2" / modnames +import icmodnames import "../dist/nimony/src/models" / nifindex_tags import typekeys import icnifcore @@ -288,9 +290,23 @@ proc toNifSymName(w: var Writer; sym: PSym): string = # Process-local backend sym (closure env field / hidden `:env` param minted # during a VM transform): re-home to the current module with the `@bk` # marker so each referencing module self-contains it. See transformBody. + # + # Use `itemId.item` (the writer's dedup identity, see `emittedBackendSyms`) + # as the numeric name component, NOT `disamb`: closure `:env` syms in one + # module are minted from TWO id spaces — the backend lower stage's + # `tb.idgen` and sem's `vmTransfIdgen` (transf.transformBody) — whose + # `disambTable`s each start `:env` at the same low count, so a macro-lowered + # `:env` (e.g. `implementSendProcBody`) and a backend-lowered one + # (`peerTrimmerHeartbeat`) collide on `:env.2.@bk`. Two distinct syms + # then share a NIF name; the loader's name-keyed index/`c.syms` return the + # first for both, so one proc's `:env` gets the OTHER proc's env type + # (mismatched-pointer C, "has no member colonup_" at link). `itemId.item` is + # unique per `@bk` sym (both are emitted as defs, see writeSym), mirroring + # how `@bk` TYPES already key off `uniqueId.item` (nifTypeName). The loader + # copies this back into `disamb` (sn.count), so `globalName` round-trips. result = sym.name.s result.add '.' - result.addInt sym.disamb + result.addInt sym.itemId.item result.add '.' result.add modname(w.currentModule, w.infos.config) result.add BackendLocalMarker @@ -496,8 +512,19 @@ proc writeTypeDef(w: var Writer; dest: var IcBuilder; typ: PType) = # name in isolation (cg seeks the `.t.nif`/`.s.nif` index entry), so its # fields must be DEFS here, not entry-deduped SymUses whose def lives # elsewhere in the `(lowered)` entry and is never read by the seek. + # + # `emittedFieldSyms` only guards against a field being def'd twice WITHIN one + # reclist, so scope it per-reclist: a generic object and its instances SHARE one + # field PSym (same itemId) yet each instance carries a DISTINCT field type (e.g. + # `MDigest[256].data: array[32,byte]` vs `MDigest[384].data: array[48,byte]`), so + # each reclist needs its OWN typed def. A Writer-global set deduped every instance + # after the first to a typeless `SymUse` stub (nil typ/owner on load → crash in + # destructor lifting). Field NIF names are local (no module suffix, not in the + # global `c.syms`), so def'ing the same field in two reclists never collides. inc w.inTypeReclist + let savedFieldSyms = move w.emittedFieldSyms writeNode(w, dest, typ.nImpl) + w.emittedFieldSyms = savedFieldSyms dec w.inTypeReclist writeSym(w, dest, typ.ownerFieldImpl) writeSym(w, dest, typ.symImpl) @@ -901,6 +928,10 @@ proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) = if n == nil: dest.addDotToken else: + if nfLazyBody in n.flags and forceLazyBodyHook != nil: + # Materialize a deferred body before serializing so its real flags/typ and + # children are written (never the empty `nfLazyBody` placeholder). + forceLazyBodyHook(n) case n.kind of nkNone: assert n.typField == nil, "nkNone should not have a type" @@ -1070,10 +1101,18 @@ proc writeToplevelNode(w: var Writer; dest, bottom: var IcBuilder; n: PNode) = of nkEmpty: discard "ignore" of nkTypeSection, nkCommentStmt, nkMixinStmt, nkBindStmt, nkUsingStmt, - nkPragma, nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef, nkTemplateDef: # We write purely declarative nodes at the bottom of the file writeNode(w, bottom, n) + of nkPragma: + # Top-level pragmas — chiefly `{.emit.}`, plus the `{.push/pop.}` that guard + # its neighbours — must survive the backend reload so the `cg` stage re-runs + # genPragma/genEmit. The bottom (implementation) section is reloaded lazily + # BY SYMBOL INDEX, which a symbol-less pragma can never be on, so a pragma + # written there is silently dropped on reload (e.g. a module-level `#include` + # vanishes and the generated C fails to compile). The header init section is + # replayed verbatim by `processTopLevel`, so write it there instead. + writeNode(w, dest, n) of nkConstSection: writeGlobals(w, bottom, n) of nkLetSection, nkVarSection: @@ -1756,8 +1795,18 @@ type semIndex: Table[string, NifIndexEntry] semTried: bool # `semBuf`/`semIndex` load attempted (idempotent) + PendingBody = object + ## A deferred routine body (bodyPos son). `cursor` points AT the body node in + ## the module buffer (kept alive by the cursor's refcounted owner); `localSyms` + ## is the snapshot of the enclosing sym def's local symbols so body-local + ## references resolve to the SAME PSyms the signature already created. + cursor: Cursor + thisModule: string + localSyms: Table[string, PSym] + DecodeContext* = object infos: LineInfoWriter + pendingBodies: Table[int, PendingBody] # nodeId(placeholder) -> deferred body #moduleIds: Table[string, int32] types: Table[string, (PType, NifIndexEntry)] syms: Table[string, (PSym, NifIndexEntry)] @@ -1767,11 +1816,60 @@ type ## Mangled module name of the module being compiled fresh (cmdM). Symbols ## belonging to it that are re-exported by a dependency must NOT be loaded ## as stubs, otherwise they collide with the freshly compiled originals. + symLoads, typeLoads: CountTable[FileIndex] + ## Diagnostics (opt-in via env `NIM_IC_LOADSTATS`): per OWNING-module count + ## of stub materializations in THIS process. Quantifies the "every backend + ## worker deserializes system.bif + a bunch of others" cost — breadth (how + ## many syms) attributed to duplication axis (which shared module). proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache) +var loadStatsInit {.threadvar.}: int # 0=unknown 1=on 2=off +var statsCtxPtr {.threadvar.}: ptr DecodeContext +var loaderCtx {.threadvar.}: ptr DecodeContext # the live `program`; for lazy-body + # materialization off the len hook +var nodesDecoded {.threadvar.}: int # all PNodes materialized this proc +var astFieldNodes {.threadvar.}: int # subset: routine-body (s.ast) subtrees + +proc dumpLoadStatsExit() {.noconv.} = + if statsCtxPtr == nil: return + let c = statsCtxPtr + var merged = initTable[FileIndex, array[2, int]]() + for m, cnt in c.symLoads.pairs: merged.mgetOrPut(m, [0, 0])[0] = cnt + for m, cnt in c.typeLoads.pairs: merged.mgetOrPut(m, [0, 0])[1] = cnt + var order: seq[FileIndex] = @[] + var totS, totT: int = 0 + for m, a in merged: + order.add m + totS += a[0]; totT += a[1] + sort(order, proc (a, b: FileIndex): int = + (merged[b][0] + merged[b][1]) - (merged[a][0] + merged[a][1])) + let params = commandLineParams() + let target = if params.len > 0: params[^1] else: "?" + stderr.writeLine "=== IC loadstats pid=" & $getCurrentProcessId() & + " main=" & c.mainModuleSuffix & " target=" & target & " ===" + stderr.writeLine " TOTAL symLoads=" & $totS & " typeLoads=" & $totT & + " modulesTouched=" & $order.len + let pct = if nodesDecoded > 0: 100 * astFieldNodes div nodesDecoded else: 0 + stderr.writeLine " PNODES decoded=" & $nodesDecoded & " routineBody=" & + $astFieldNodes & " (" & $pct & "% deferrable via lazy PSym.ast)" + for m in order: + let a = merged[m] + let name = if c.mods.hasKey(m): c.mods[m].suffix else: "?" + stderr.writeLine " " & $(a[0] + a[1]) & "\tsym=" & $a[0] & " typ=" & $a[1] & + "\t" & name + +proc recordLoad(c: var DecodeContext; m: FileIndex; isType: bool) = + if loadStatsInit == 0: + loadStatsInit = if existsEnv("NIM_IC_LOADSTATS"): 1 else: 2 + if loadStatsInit == 1: + statsCtxPtr = addr c + addExitProc(dumpLoadStatsExit) + if loadStatsInit == 2: return + if isType: c.typeLoads.inc(m) else: c.symLoads.inc(m) + proc nextBackendSymItem*(c: var DecodeContext; module: int32): int32 = ## Allocate the next backend-minted SYM item for `module` from the SAME ## per-module counter the loader uses when it re-homes `@bk` syms loaded from @@ -2002,6 +2100,42 @@ proc reconstructSysType(c: var DecodeContext; name: string; k: int; itemVal: int result.alignImpl = int16 c.infos.config.target.ptrSize c.types[name] = (result, NifIndexEntry()) +proc stripBkSuffix(rawMod: string): (bool, string) {.inline.} = + ## Split a possibly-`@bk` (BackendLocalMarker) module suffix into + ## `(isBackendMinted, realSuffix)`. See `toNifSymName`/`nifTypeName`. + if rawMod.endsWith(BackendLocalMarker): + (true, rawMod[0 ..< rawMod.len - BackendLocalMarker.len]) + else: + (false, rawMod) + +proc nextSymId(c: var DecodeContext; module: FileIndex; isBk: bool): ItemId = + ## Mint the next per-module SYM id from `symCounter`: a `backendItemId` for a + ## process-local `@bk` sym, else a plain loader `itemId`. Both draw from the one + ## counter so loaded and cg-minted backend syms stay disjoint (see + ## `nextBackendSymItem`). Types do NOT use this — they preserve the item parsed + ## from their own name (see `tryCreateTypeStub`). + let val = addr c.mods[module].symCounter + inc val[] + result = if isBk: backendItemId(module.int32, val[]) else: itemId(module.int32, val[]) + +proc mintSymId(c: var DecodeContext; rawMod: string): (FileIndex, ItemId) = + ## Resolve a possibly-`@bk` module suffix to its FileIndex and mint a fresh sym + ## id for it — the common case where the module is not needed before minting + ## (see `stripBkSuffix`/`nextSymId`). + let (isBk, realMod) = stripBkSuffix(rawMod) + let module = moduleId(c, realMod) + result = (module, c.nextSymId(module, isBk)) + +proc makePartialSymStub(c: var DecodeContext; symAsStr: string; sn: ParsedSymName; + id: ItemId; entry: NifIndexEntry): PSym = + ## Create + cache (keyed by the NIF name) a `Partial` global-sym stub, lazily + ## filled later by `loadSym` from `entry`. `stubKindAndName` strips NIF-only + ## markers (e.g. a package's `PkgMarker`) so the backend mangles the clean name. + let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) + result = PSym(itemId: id, kindImpl: stubKind, name: stubName, + disamb: sn.count.int32, state: Partial) + c.syms[symAsStr] = (result, entry) + proc tryCreateTypeStub(c: var DecodeContext; name: string): PType = ## Like `createTypeStub` but returns nil instead of raising when the type has ## no offset in its module index (used by the best-effort `(offer …)` loader). @@ -2024,8 +2158,7 @@ proc tryCreateTypeStub(c: var DecodeContext; name: string): PType = let suffix = name.substr(i) if suffix == SysModuleSuffix: return reconstructSysType(c, name, k, itemVal) - let isBk = suffix.endsWith(BackendLocalMarker) - let realSuffix = if isBk: suffix[0 ..< suffix.len - BackendLocalMarker.len] else: suffix + let (isBk, realSuffix) = stripBkSuffix(suffix) let modIdx = moduleId(c, realSuffix).int32 let id = if isBk: backendItemId(modIdx, itemVal) else: itemId(modIdx, itemVal) let modFi = id.module.FileIndex @@ -2037,34 +2170,12 @@ proc tryCreateTypeStub(c: var DecodeContext; name: string): PType = c.types[name] = (result, c.mods[modFi].index.getOrDefault(name)) proc createTypeStub(c: var DecodeContext; name: string): PType = + ## As `tryCreateTypeStub`, but a missing index offset is a hard error (the + ## caller demanded a definition that must exist). assert name.startsWith("`t") - result = c.types.getOrDefault(name)[0] + result = tryCreateTypeStub(c, name) if result == nil: - var i = len("`t") - var k = 0 - while i < name.len and name[i] in {'0'..'9'}: - k = k * 10 + name[i].ord - ord('0') - inc i - if i < name.len and name[i] == '.': inc i - var itemVal = 0'i32 - while i < name.len and name[i] in {'0'..'9'}: - itemVal = itemVal * 10'i32 + int32(name[i].ord - ord('0')) - inc i - if i < name.len and name[i] == '.': inc i - let suffix = name.substr(i) - if suffix == SysModuleSuffix: - return reconstructSysType(c, name, k, itemVal) - let isBk = suffix.endsWith(BackendLocalMarker) - let realSuffix = if isBk: suffix[0 ..< suffix.len - BackendLocalMarker.len] else: suffix - let modIdx = moduleId(c, realSuffix).int32 - let id = if isBk: backendItemId(modIdx, itemVal) else: itemId(modIdx, itemVal) - let modFi = id.module.FileIndex - if not hasTypeOffset(c, modFi, name): - raiseAssert "symbol has no offset: " & name - result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) - # `loadType` re-resolves the buffer via `typeCursor`, so the cached entry is a - # don't-care for types — store the primary one if any (else a 0-offset stub). - c.types[name] = (result, c.mods[modFi].index.getOrDefault(name)) + raiseAssert "symbol has no offset: " & name proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]) = @@ -2093,9 +2204,7 @@ proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: s # Local symbol - create stub and immediately load it fully # since local symbols have no index offsets for lazy loading let module = moduleId(c, thisModule) - let val = addr c.mods[module].symCounter - inc val[] - let id = itemId(module.int32, val[]) + let id = c.nextSymId(module, isBk = false) # `stubKindAndName` strips NIF-only markers (e.g. a field's `` `f ``) so the # backend mangles the clean name; `loadSymFromCursor` then fills the real kind. let (_, stubName) = stubKindAndName(c.cache, sn.name) @@ -2148,11 +2257,9 @@ proc loadFieldStub(c: var DecodeContext; symAsStr: string; thisModule: string; let sn = parseSymName(symAsStr) let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) let module = moduleId(c, thisModule) - let val = addr c.mods[module].symCounter - inc val[] # `sn.count` is the field POSITION (see toNifSymName): tuple element access reads # it directly off this stub, so preserve it. Named-object uses re-navigate by name. - result = PSym(itemId: itemId(module.int32, val[]), kindImpl: stubKind, + result = PSym(itemId: c.nextSymId(module, isBk = false), kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Complete) result.positionImpl = sn.count.int32 if typ != nil: result.typImpl = typ @@ -2177,16 +2284,9 @@ proc loadSymStub(c: var DecodeContext; symAsStr: string; thisModule: string; result = c.syms.getOrDefault(symAsStr)[0] if result == nil: # A process-local backend sym (closure env field / `:env` param) is named - # `…@bk`: home it to that module with a backendItemId so it - # stays disjoint from the loader's real per-module id space (see toNifSymName). - let isBk = sn.module.endsWith(BackendLocalMarker) - let realMod = if isBk: sn.module[0 ..< sn.module.len - BackendLocalMarker.len] - else: sn.module - let module = moduleId(c, realMod) - let val = addr c.mods[module].symCounter - inc val[] - let id = if isBk: backendItemId(module.int32, val[]) else: itemId(module.int32, val[]) - + # `…@bk`: `mintSymId` homes it to that module with a + # backendItemId so it stays disjoint from the loader's real id space. + let (module, id) = c.mintSymId(sn.module) let offs = c.mods[module].index.getOrDefault(symAsStr) if offs.offset == 0: # Only module/package self-syms are never written as `(sd)` entries, so a @@ -2199,9 +2299,7 @@ proc loadSymStub(c: var DecodeContext; symAsStr: string; thisModule: string; infoImpl: newLineInfo(module, 1, 1), state: Complete) c.syms[symAsStr] = (result, NifIndexEntry()) return result - let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) - result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) - c.syms[symAsStr] = (result, offs) + result = c.makePartialSymStub(symAsStr, sn, id, offs) proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PSym = @@ -2306,6 +2404,7 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms proc loadType*(c: var DecodeContext; t: PType) = if t.state != Partial: return t.state = c.loadedState + recordLoad(c, t.itemId.module.FileIndex, isType = true) # A backend-minted (`@bk`) closure-env type produced by the `lower` stage lives # ONLY in the `.t.nif` and is keyed by its `@bk` name (see nifTypeName), not the # canonical `typeToNifSym` (which asserts non-`@bk`). Reconstruct that name so a @@ -2399,7 +2498,10 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: s.ownerFieldImpl = loadSymStub(c, n, thisModule, localSyms) # Load the AST for routine symbols and constants # Constants need their AST for astdef() to return the constant's value + let astNodesBefore = nodesDecoded s.astImpl = loadNode(c, n, thisModule, localSyms) + if loadStatsInit == 1 and s.kindImpl in routineKinds: + astFieldNodes += nodesDecoded - astNodesBefore loadLoc c, n, s.locImpl s.constraintImpl = loadNode(c, n, thisModule, localSyms) s.instantiatedFromImpl = loadSymStub(c, n, thisModule, localSyms) @@ -2426,6 +2528,8 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: proc loadSym*(c: var DecodeContext; s: PSym) = if s.state != Partial: return s.state = c.loadedState + if loaderCtx == nil: loaderCtx = addr c + recordLoad(c, s.itemId.module.FileIndex, isType = false) let symsModule = s.itemId.module.FileIndex let nifname = globalName(s, c.infos.config) var n = cursorFromIndexEntry(c, symsModule, c.syms[nifname][1]) @@ -2478,6 +2582,7 @@ template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNod proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PNode = + if loadStatsInit == 1: inc nodesDecoded result = nil case n.kind of Symbol: @@ -2544,9 +2649,7 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; if sym == nil: # First time seeing this local symbol - create it let module = moduleId(c, thisModule) - let val = addr c.mods[module].symCounter - inc val[] - let id = itemId(module.int32, val[]) + let id = c.nextSymId(module, isBk = false) # strip NIF-only markers (a field's `` `f ``) so the backend sees the # clean name; `loadSymFromCursor` below fills the real kind. let (_, stubName) = stubKindAndName(c.cache, sn.name) @@ -2587,9 +2690,7 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; else: sym = c.syms.getOrDefault(symName)[0] if sym == nil: - let val = addr c.mods[m].symCounter - inc val[] - sym = PSym(itemId: itemId(m.int32, val[]), kindImpl: skStub, + sym = PSym(itemId: c.nextSymId(m, isBk = false), kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[symName] = (sym, NifIndexEntry()) @@ -2666,6 +2767,27 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; of nkNilLit: c.withNode n, result, kind: discard + of routineDefs: + # Defer the heavy `bodyPos` son: build the routine-def header eagerly, but + # install a `nfLazyBody` placeholder (carrying the real body kind, so cheap + # `ast[bodyPos].kind != nkEmpty` checks need no load) whose children are + # materialized on demand (see `materializeLazyBody`, driven by the `len` + # hook). An empty body is a single node — not worth deferring. + c.withNode n, result, kind: + var idx = 0 + while n.hasMore: + if idx == bodyPos and n.kind == TagLit and + n.nodeKind notin {nkEmpty, nkNone}: + let info = c.infos.oldLineInfo(n.info, cursorPool(n)) + let ph = newNodeI(n.nodeKind, info) + ph.flags.incl nfLazyBody + c.pendingBodies[cast[int](ph)] = + PendingBody(cursor: n, thisModule: thisModule, localSyms: localSyms) + result.sons.add ph + skip n + else: + result.sons.add c.loadNode(n, thisModule, localSyms) + inc idx else: c.withNode n, result, kind: while n.hasMore: @@ -2673,25 +2795,46 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; else: raiseAssert "expected string literal but got " & $n.kind +proc materializeLazyBody*(c: var DecodeContext; node: PNode) = + ## Fill a `nfLazyBody` placeholder's children in place (identity-preserving: + ## callers already hold `node`). Decodes the deferred body from the stashed + ## cursor with the enclosing def's `localSyms` so param/local refs resolve to + ## the SAME PSyms the signature created. + node.flags.excl nfLazyBody # clear first: the loadNode below calls `len` + let key = cast[int](node) + var pb = PendingBody() + if not c.pendingBodies.pop(key, pb): return + var cur = pb.cursor + let real = c.loadNode(cur, pb.thisModule, pb.localSyms) + # `real` has the same kind as the placeholder (peeked at defer time); graft its + # decoded content onto the node the callers hold. + node.sons = real.sons + node.typField = real.typField + node.flags = real.flags + +forceLazyBodyHook = proc (n: PNode) {.nimcall, raises: [], tags: [], gcsafe.} = + # `len` (the sole caller path) MUST stay effect-free, so this hook is typed + # `raises: []`. The underlying `loadNode` chain infers `raises: [KeyError]` + # (index/sym Table lookups), but materialization only ever runs for a body + # DEFERRED during THIS load — the buffer/index is present by construction, so a + # KeyError here means a corrupt cache: a fatal bug, not a recoverable error. + # Treat it as effect-free (a `Defect`-like invariant) via a scoped cast. + if loaderCtx != nil: + {.cast(raises: []).}: + {.cast(tags: []).}: + {.cast(gcsafe).}: + materializeLazyBody(loaderCtx[], n) + proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; nifName: string; entry: NifIndexEntry; thisModule: string): PSym = ## Loads a symbol from the NIF index entry using the entry directly. ## Creates a symbol stub without looking up in the index (since the index may be moved out). result = c.syms.getOrDefault(nifName)[0] if result == nil: - let symAsStr = nifName - let sn = parseSymName(symAsStr) + let sn = parseSymName(nifName) let rawMod = if sn.module.len > 0: sn.module else: thisModule - let isBk = rawMod.endsWith(BackendLocalMarker) - let realMod = if isBk: rawMod[0 ..< rawMod.len - BackendLocalMarker.len] else: rawMod - let symModule = moduleId(c, realMod) - let val = addr c.mods[symModule].symCounter - inc val[] - - let id = if isBk: backendItemId(symModule.int32, val[]) else: itemId(symModule.int32, val[]) - let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) - result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) - c.syms[symAsStr] = (result, entry) + let (_, id) = c.mintSymId(rawMod) + result = c.makePartialSymStub(nifName, sn, id, entry) proc extractBasename(nifName: string): string = ## Extract the base name from a NIF name (ident.disamb.module -> ident) @@ -2790,9 +2933,7 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo let sn = parseSymName(symAsStr) if sn.module.len == 0: return nil # Local symbols shouldn't be hooks - let isBk = sn.module.endsWith(BackendLocalMarker) - let realMod = if isBk: sn.module[0 ..< sn.module.len - BackendLocalMarker.len] - else: sn.module + let (isBk, realMod) = stripBkSuffix(sn.module) let module = moduleId(c, realMod) # Look up the symbol in the module's index # Try both formats: with module suffix (e.g., "foo.0.modulename") and without (e.g., "foo.0.") @@ -2806,12 +2947,9 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo return nil if not alsoConsiderPrivate and offs.vis == Hidden: return nil - # Create a stub symbol - let val = addr c.mods[module].symCounter - inc val[] - let id = if isBk: backendItemId(int32(module), val[]) else: itemId(int32(module), val[]) - result = PSym(itemId: id, kindImpl: skProc, name: c.cache.getIdent(sn.name), - disamb: sn.count.int32, state: Partial) + # Create a stub symbol (skProc: `resolveSym` only resolves hook/routine syms). + result = PSym(itemId: c.nextSymId(module, isBk), kindImpl: skProc, + name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, offs) proc resolveHookSym*(c: var DecodeContext; name: string): PSym = @@ -3115,9 +3253,14 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag] skip cur elif tagIs(cur, "implementation"): cont = false - elif LoadFullAst in flags or tagIs(cur, toNifTag(nkLetSection)) or tagIs(cur, toNifTag(nkVarSection)): + elif LoadFullAst in flags or tagIs(cur, toNifTag(nkLetSection)) or + tagIs(cur, toNifTag(nkVarSection)) or tagIs(cur, toNifTag(nkPragma)): # Parse the full statement. let/var sections are loaded unconditionally # (see above) so `{.compileTime.}` globals reach the eager initializer. + # Top-level pragmas are loaded too: a module-level `{.emit.}` (and the + # `{.push/pop.}` around it) must reach the `cg` stage's genPragma/genEmit, + # else e.g. a `#include` is dropped and the generated C won't compile. + # writeToplevelNode routes these into this header section. let stmtNode = loadNode(c, cur, suffix, localSyms) if stmtNode != nil: result.topLevel.sons.add stmtNode diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 63de1814fb..d2af6a9064 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -339,6 +339,14 @@ type # because openSym experimental switch is disabled # gives warning instead nfLazyType # node has a lazy type + nfLazyBody # IC: this node is a placeholder for a routine body (bodyPos son) + # not yet materialized. Reading its children (via `len`/`safeLen`) + # triggers `forceLazyBodyHook`. Process-local, stripped on serialize. + nfBroadcast # this `nkBracket` is a *broadcast* default array: a single son + # standing for `lengthOrd` identical zero copies (see + # `broadcastArrayThreshold`). The flag disambiguates it from an + # ordinary 1-element collection (e.g. a seq value that happens to + # carry an array type), so it must survive copies + serialization. TNodeFlags* = set[TNodeFlag] TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 47) @@ -866,7 +874,8 @@ const nfFromTemplate, nfDefaultRefsParam, nfExecuteOnReload, nfLastRead, nfFirstWrite, nfSkipFieldChecking, - nfDisabledOpenSym, nfLazyType} + nfDisabledOpenSym, nfLazyType, + nfBroadcast} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 @@ -903,7 +912,24 @@ const defaultOffset* = -1 +var forceLazyBodyHook*: proc (n: PNode) {.nimcall, raises: [], tags: [], gcsafe.} + ## Set by the IC loader (ast2nif). When a node carries `nfLazyBody`, any access + ## to its children through `len` materializes the deferred routine body in place. + ## `safeLen` delegates to `len`, so it is covered transitively; a lazy body is + ## never a leaf kind, so the `{nkNone..nkNilLit}` short-circuit never hides it. + ## + ## The type MUST be effect-free (`raises: []`/`tags: []`): `len` is a fundamental + ## `PNode` accessor that the whole compiler — and every compiler-as-library + ## consumer (nimble, nimsuggest, ...) — assumes cannot raise. An unannotated + ## `proc` var defaults to `raises: [Exception]`, so the indirect call tainted + ## `len`/`safeLen`/`items` with `Exception`, breaking any iterator/`{.raises.}` + ## over a `PNode` (e.g. nimble's `extract {.raises: [CatchableError].}`). + ## Materialization is a pure in-memory buffer transform; a corrupt buffer is a + ## `Defect` (`raiseAssert`), which is outside exception tracking. + proc len*(n: PNode): int {.inline.} = + if nfLazyBody in n.flags and forceLazyBodyHook != nil: + forceLazyBodyHook(n) result = n.sons.len proc safeLen*(n: PNode): int {.inline.} = diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index df50d93954..e023eeee7b 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3964,6 +3964,13 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo; result: var Builder) let elemTyp = skipTypes(t.elementType, abstractRange+{tyOwned}-{tyTypeDesc}) if isOpaqueImportcType(elemTyp): result.add "{0}" + elif toInt(lengthOrd(p.config, t.indexType)) > broadcastArrayThreshold and + elemTyp.kind in {tyInt..tyUInt64, tyBool, tyChar, tyFloat..tyFloat128, + tyPtr, tyPointer, tyCstring}: + # Large array of a scalar whose default is the zero representation: a single + # C `{0}` zero-fills all `lengthOrd` slots instead of emitting that many + # initializers (keeps huge SSZ-style zero buffers compact in the C output). + result.add "{0}" else: var arrInit: StructInitializer result.addStructInitializer(arrInit, kind = siArray): @@ -4250,7 +4257,13 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; resul var d: TLoc = initLocExpr(p, n) result.add rdLoc(d) of tyArray, tyVarargs: - genConstSimpleList(p, n, isConst, result) + if isDefaultBroadcastArray(n, p.config): + # Compact zero/null-default array (see `isDefaultBroadcastArray`): the + # whole thing is the null value of every slot, so a single C `{0}` + # zero-fills all `lengthOrd` elements — no need to materialise them. + result.add "{0}" + else: + genConstSimpleList(p, n, isConst, result) of tyTuple: genConstTuple(p, n, isConst, typ, result) of tyOpenArray: diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 98a85b38ae..cd2e325dee 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -819,7 +819,11 @@ proc genRecordFieldsAux(m: BModule; n: PNode, # don't use fieldType here because we need the # tyGenericInst for C++ template support let noInit = sfNoInit in field.flags or (field.typ.sym != nil and sfNoInit in field.typ.sym.flags) - if not noInit and (fieldType.isOrHasImportedCppType() or hasCppCtor(m, field.owner.typ)): + # Under `nim ic`, object fields are local NIF syms restored without an + # `owner`; `rectype` is the owning record type, so fall back to it rather + # than deref a nil `field.owner`. + let ownerTyp = if field.owner != nil: field.owner.typ else: rectype + if not noInit and (fieldType.isOrHasImportedCppType() or hasCppCtor(m, ownerTyp)): var didGenTemp = false initializer = genCppInitializer(m, nil, fieldType, didGenTemp) result.addField(field, sname, typ, isFlexArray, initializer) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index b1f6f862c7..52c8726485 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1663,7 +1663,15 @@ proc genProcPrototype(m: BModule, sym: PSym) = useHeader(m, sym) if lfNoDecl in sym.loc.flags or sfCppMember * sym.flags != {}: return if lfDynamicLib in sym.loc.flags: - if sym.itemId.module != m.module.position and + if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg": + # Under IC per-module cg every demander emits the dynlib proc's DEFINITION + # locally (findPendingModule returns `m`, so symInDynamicLib follows this + # call and the merge stage keeps one def per C name). Emitting the + # cross-module `extern` proto here would register `sym.id` in + # `m.declaredThings` and thereby make that `symInDynamicLib` skip, leaving + # the `Dl_*` symbol declared-but-never-defined -> undefined at link. + discard "definition emitted by symInDynamicLib" + elif sym.itemId.module != m.module.position and not containsOrIncl(m.declaredThings, sym.id): let vis = if isReloadable(m, sym): StaticProc else: Extern let name = mangleDynLibProc(sym) diff --git a/compiler/deps.nim b/compiler/deps.nim index f95a884da9..3e09a5e40f 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -15,7 +15,7 @@ import options, msgs, lineinfos, pathutils, condsyms, modulepaths, extccomp, cnif, platform import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder] -import "../dist/nimony/src/gear2" / modnames +import icmodnames import icnifcore type @@ -956,7 +956,12 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string result = nimcache / c.nodes[0].files[0].modname & ".backend.build.nif" let mainNif = c.nodes[0].files[0].nimFile - let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt) + # Honor `--out`/`--outdir`: `cmdIc`'s `setOutFile` populated `conf.outFile` + # (the user's `--out`, or the default ``), so `absOutFile` is + # the final link target — exactly what a whole-program `nim c` would produce. + # The `link` child computes its own output from its project name, so the path + # is also forwarded to it below. + let exeFile = string(c.config.absOutFile) let mergeFile = nimcache / MergeDecisionFile # Per-node output paths. @@ -1117,6 +1122,11 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string b.addIdent "nim_nifc" b.withTree "args": b.addStrLit "--icBackendStage:link" + # The link child is its own `cmdNifC` process whose project is the main + # module, so it would default the binary to `/
`. + # Forward the resolved target so it writes exactly `exeFile` (`--out`'s + # path splits back into outDir+outFile in the child). + b.addStrLit "--out:" & exeFile for i in 0 ..< c.nodes.len: if live[i]: inputStr cFiles[i] outputStr exeFile diff --git a/compiler/ic/enum2nif.nim b/compiler/ic/enum2nif.nim index 4b7860fc96..b28a4ae4d5 100644 --- a/compiler/ic/enum2nif.nim +++ b/compiler/ic/enum2nif.nim @@ -1473,6 +1473,8 @@ proc genFlags*(s: set[TNodeFlag]; dest: var string) = of nfSkipFieldChecking: dest.add "s0" of nfDisabledOpenSym: dest.add "d3" of nfLazyType: dest.add "l1" + of nfLazyBody: discard # process-local placeholder; never serialized + of nfBroadcast: dest.add "v" proc parse*(t: typedesc[TNodeFlag]; s: string): set[TNodeFlag] = @@ -1533,6 +1535,7 @@ proc parse*(t: typedesc[TNodeFlag]; s: string): set[TNodeFlag] = inc i else: result.incl nfSem of 't': result.incl nfTransf + of 'v': result.incl nfBroadcast of 'w': result.incl nfFirstWrite else: discard inc i diff --git a/compiler/icmodnames.nim b/compiler/icmodnames.nim new file mode 100644 index 0000000000..a162da834e --- /dev/null +++ b/compiler/icmodnames.nim @@ -0,0 +1,55 @@ +# +# +# The Nim Compiler +# (c) Copyright 2026 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim's OWN module-suffix, replacing nimony's `gear2/modnames.moduleSuffix`. +## +## nimony's version hashes a path made RELATIVE to `getCurrentDir()` (or the +## shortest search-path-relative form), so the produced suffix depends on the +## current working directory AND the searchPath set. Under `nim ic` the +## DISCOVERY pass (`deps.nim`, in the driver process) and the COMPILE pass +## (`nifgen`/`typekeys`, in a child `nim m` process) can run with different CWDs +## or `--path` sets, so the SAME file hashes to two different suffixes: e.g. +## `std/staticos` became `sta5rk8sn1` at discovery but `sta4c0qxk` at compile, so +## every importer waited forever for a `.s.bif` that was actually written under +## the other name — a cold `nim ic` build (of anything pulling in `std/os`, whose +## `oscommon` does `from std/staticos import PathComponent`) never converged. +## +## Hashing the CANONICAL ABSOLUTE path makes the suffix a pure function of the +## file, identical across every process and call site. The base-name prefix + +## base-36 `uhash` layout is kept byte-for-byte compatible with the old scheme so +## nothing but the hashed string changes. + +import std/os +import "../dist/nimony/src/lib" / tinyhashes + +const + PrefixLen = 3 # keep it short: the suffix ends up in every mangled C name + Base36 = "0123456789abcdefghijklmnopqrstuvwxyz" + +proc moduleSuffix*(path: string; searchPaths: openArray[string]): string = + ## `searchPaths` is accepted for signature-compatibility with the replaced + ## `modnames.moduleSuffix` but is deliberately IGNORED — the suffix must not + ## depend on the search-path set or the CWD (see the module doc). + # Absolute inputs (the norm at every call site: `toFullPath`/`projectFull`) + # pass straight through `normalizedPath` with no `getCurrentDir` involvement; + # a stray relative path is made absolute against the CWD only as a fallback. + var f = path + if not isAbsolute(f): + try: f = absolutePath(f) + except CatchableError: discard + f = normalizedPath(f) + let m = splitFile(f).name + var id = uhash(f) + result = newStringOfCap(10) + for i in 0 ..< min(m.len, PrefixLen): + result.add m[i] + # base-36 of the hash, low digit first (order is irrelevant for identity). + while id > 0'u32: + result.add Base36[int(id mod 36'u32)] + id = id div 36'u32 diff --git a/compiler/main.nim b/compiler/main.nim index fbc2b81be6..efbf496d4f 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -436,6 +436,9 @@ proc mainCommand*(graph: ModuleGraph) = # Generate .build.nif for nifmake setUseIc(true) wantMainModule(conf) + # Resolve the output binary path (honoring `--out`) up front, like cmdNifC: + # the backend build file derives the link target from `conf.absOutFile`. + setOutFile(conf) when not defined(nimKochBootstrap): commandIc(conf) else: diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 0b238189c8..78a5f42ab3 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -906,6 +906,10 @@ proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool = proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} = result = s.ast[bodyPos] + if result != nil and nfLazyBody in result.flags and forceLazyBodyHook != nil: + # Sanctioned body-access gate (see astdef.bodyPos): materialize the deferred + # IC body so callers may safely touch `.sons` directly, not only via `len`. + forceLazyBodyHook(result) assert result != nil when not defined(nimKochBootstrap): diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index b23f5d2291..b77f3cdbd3 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -24,12 +24,22 @@ when defined(nimPreviewSlimSystem): import ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif, typekeys, - cnif + cnif, icmodnames from cgmeth import generateIfMethodDispatchers from transf import transformBody from injectdestructors import injectDestructorCalls import ic / replayer +proc systemNifSuffix(conf: ConfigRef): string = + ## The system module's NIF suffix, derived from `system.nim`'s path EXACTLY as + ## the frontend derives it (deps.nim's `toPair` on `libpath/system.nim`), so the + ## backend loads the very `.s.bif` the frontend wrote. It must NOT be a constant: + ## `moduleSuffix` (icmodnames) now hashes the absolute path, so the system suffix + ## is install-dependent (was hardcoded `sysma2dyk`, valid only for the old + ## relative-path scheme where `system.nim` always relativized to `system.nim`). + moduleSuffix((conf.libpath / RelativeFile"system.nim").string, + cast[seq[string]](conf.searchPaths)) + proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex; nifFiles: var seq[string]; depFlags: set[LoadFlag] = {LoadFullAst}): seq[PrecompiledModule] = @@ -198,8 +208,16 @@ proc ownsRuntimeRoutine(s: PSym; modPos: int): bool = {sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and s.typ != nil and not signatureHasMetaType(s.typ) and s.ast != nil and s.ast.safeLen > bodyPos and - s.ast[genericParamsPos].kind == nkEmpty and - s.ast[bodyPos].kind != nkEmpty + s.ast[genericParamsPos].kind == nkEmpty + # NOTE: an `nkEmpty` body is NOT a disqualifier. A concrete, owned, non- + # forward/-importc/-magic routine whose body folds to nothing is still a real + # definition the owner must emit (`void f(void){}`), exactly as whole-program + # cgen does — else a cross-module caller links to nothing. This bites e.g. + # Nimbus' `extras.incInternalErrors`, a plain `proc` whose sole statement is a + # metrics-counter `.inc()` that the `metrics` library expands to a no-op when + # the importing tool (ncli) builds with `-u:metrics`; the body is then a bare + # `nkEmpty`, but `state_transition_epoch` still calls it. Forward declarations + # (the other empty-body case) carry `sfForward` and are excluded above. proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) = ## Generate C code for a single module. @@ -245,7 +263,7 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex): ## and only needs each module's `(replay ...)` directives, which load anyway. resetForBackend(g) var isKnownFile = false - let systemFileIdx = registerNifSuffix(g.config, "sysma2dyk", isKnownFile) + let systemFileIdx = registerNifSuffix(g.config, systemNifSuffix(g.config), isKnownFile) g.config.m.systemFileIdx = systemFileIdx var precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface}) g.systemModule = precompSys.module @@ -268,7 +286,7 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex): # closure here too — otherwise `findTargetModule` cannot resolve their suffix. block: var visited = initHashSet[string]() - visited.incl "sysma2dyk" + visited.incl systemNifSuffix(g.config) for m in modules: visited.incl cachedModuleSuffix(g.config, FileIndex m.module.position) var stack: seq[ModuleSuffix] = @[] @@ -311,14 +329,14 @@ proc loadDepClosure(g: ModuleGraph; targetSuffix: string): ## dispatchers, runs essentially alone since every other `.c.nif` precedes it). resetForBackend(g) var isKnownFile = false - let systemFileIdx = registerNifSuffix(g.config, "sysma2dyk", isKnownFile) + let systemFileIdx = registerNifSuffix(g.config, systemNifSuffix(g.config), isKnownFile) g.config.m.systemFileIdx = systemFileIdx let precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface}) g.systemModule = precompSys.module var modules: seq[PrecompiledModule] = @[] var visited = initHashSet[string]() - visited.incl "sysma2dyk" + visited.incl systemNifSuffix(g.config) # Only the target is codegen'd, so only it needs its full AST; the closure is # loaded interface-only (demanded bodies come lazily from the kept-open @@ -730,31 +748,33 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) = let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx) let targetIsMain = g.config.icBackendModule.len == 0 or g.config.icBackendModule == mainSuffix - var modules: seq[PrecompiledModule] - var precompSys: PrecompiledModule - var target: PrecompiledModule - if targetIsMain: - var nifFiles: seq[string] - (modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx) - if modules.len == 0: - rawMessage(g.config, errGenerated, - "Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx)) - return - target = findTargetModule(g, modules, precompSys, g.config.icBackendModule) - else: - (modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule) - if target.module == nil: + # emit renders a module's final `.c` PURELY from its own `.c.nif` and the merge + # decision (see `renderCFromArtifact` — text filtering, no AST is touched). It + # used to load the target's whole transitive import closure as BModules solely + # to reach `getCFile(bmod)` for the output path. Under the fire-all-every-edit + # merge barrier (every `emit` re-fires whenever `merge` bumps the decision's + # mtime — deliberate insurance so a decision change re-renders all `.c` + # consistently) that per-process `loadDepClosure` was the bulk of a warm + # rebuild's cost: 240 processes each re-parsing a module closure only to filter + # a handful of `.c.nif`s whose bytes are usually unchanged. Derive the `.c` + # path directly instead — the SAME pure computation `deps.nim.backendCFile` + # uses to DECLARE this stage's output (`getCFile` == that formula) — so an emit + # process loads nothing and the fire-all costs process-startup, not a graph load. + let cfilename = + if targetIsMain: AbsoluteFile toFullPath(g.config, mainFileIdx) + else: AbsoluteFile g.config.icBackendModule + let cfile = changeFileExt(completeCfilePath(g.config, + mangleModuleName(g.config, cfilename).AbsoluteFile), ".nim.c").string + let artifact = cfile & ".nif" + if not fileExists(artifact): rawMessage(g.config, errGenerated, - "per-module emit: module not found for suffix: " & g.config.icBackendModule) + "per-module emit: missing .c.nif artifact for suffix: " & g.config.icBackendModule) return let decision = readMergeDecision(getNimcacheDir(g.config).string / MergeDecisionFile) if decision.broken: rawMessage(g.config, errGenerated, "per-module emit: missing or unparsable merge decision " & MergeDecisionFile) return - let bmod = BModuleList(g.backend).mods[target.module.position] - let cfile = getCFile(bmod).string - let artifact = cfile & ".nif" var dropped = 0 let code = renderCFromArtifact(artifact, decision, extractFilename(artifact), dropped) # Write the `.c` content-stably. `merge` re-runs on any edit and bumps the diff --git a/compiler/nifgen.nim b/compiler/nifgen.nim index 8467a93473..76e32e59c3 100644 --- a/compiler/nifgen.nim +++ b/compiler/nifgen.nim @@ -16,7 +16,7 @@ import import "../dist/nimony/src/lib" / nifbuilder import "../dist/nimony/src/models" / nifler_tags -import "../dist/nimony/src/gear2" / modnames +import icmodnames ## This was copied from Nifler's bridge.nim. However, this code will evolve ## in a different direction as it needs to translate the semchecked AST which diff --git a/compiler/options.nim b/compiler/options.nim index 14521d8a59..d16c42904d 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -29,7 +29,7 @@ const nimEnableCovariance* = defined(nimEnableCovariance) - icFormatVersion* = "25" + icFormatVersion* = "29" ## Version of the IC cache format (the sem-NIF module layout written by ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever ## that layout changes: `commandIc` wipes a nimcache whose `ic.version` diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 42285618ff..371cf539f7 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -476,7 +476,12 @@ proc foldArrayAccess(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNo #localError(g.config, n.info, formatErrorIndexBound(idx, x.len-1) & $n) of nkBracket: idx -= toInt64(firstOrd(g.config, x.typ)) - if idx >= 0 and idx < x.len: result = x[int(idx)] + if isDefaultBroadcastArray(x, g.config): + # compact default array: any in-bounds index folds to the default element + if idx >= 0 and idx < toInt64(lengthOrd(g.config, x.typ.skipTypes(abstractInst))): + result = copyTree(x[0]) + else: result = nil + elif idx >= 0 and idx < x.len: result = x[int(idx)] else: result = nil #localError(g.config, n.info, formatErrorIndexBound(idx, x.len-1) & $n) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 73e3b4072a..05e24894a6 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1126,9 +1126,20 @@ proc trackCall(tracked: PEffects; n: PNode) = # the per-module IC backend emits every owned routine (no DCE) and would # otherwise feed the magic to codegen. Mirrors the `tfTriggersCompileTime -> # sfCompileTime` path in `semProcAux`. + # + # GATE TO THE IC STAGES ONLY (`cmdM` sem + `cmdNifC` cg). The magic can reach a + # runtime proc's body via an INLINED TEMPLATE (not a macro/template *owner*, so + # the `insideMeta` walk below can't see it) — e.g. confutils' runtime + # `addConfigFile`/json-serialization's `inputFile` expand a serialization + # template that pastes a `getAst`/`quote` magic inline. Under plain `nim c` such + # a proc still code-generates fine (the magic folds / is demand-pruned), so + # marking it `sfCompileTime` there is a pure regression: "request to generate + # code for .compileTime proc". Only the emit-everything IC backend needs the + # mark, so restrict it to `{cmdM, cmdNifC}` (was `!= cmdNimscript`, which + # wrongly swept in `cmdCompileToC`/JS/`cmdCheck`). if a.kind == nkSym and a.sym.magic in {mNLen..mNError, mSlurp..mQuoteAst} and tracked.owner != nil and tracked.owner.kind in routineKinds and - tracked.config.cmd != cmdNimscript and tracked.inNimvmBranch == 0: + tracked.config.cmd in {cmdM, cmdNifC} and tracked.inNimvmBranch == 0: # ...but NOT under `nim e`: nimscript has no codegen backend to protect, and # marking a routine `sfCompileTime` makes `semExpr` eagerly fold calls to it # at sem time (emConst), where module-level globals it reads have no VM slot @@ -1198,8 +1209,16 @@ proc trackCall(tracked: PEffects; n: PNode) = else: if laxEffects notin tracked.c.config.legacyFeatures and a.kind == nkSym and a.sym.kind in routineKinds: - let (isHook, opKind) = findHookKind(a.sym.name.s) - if (not isHook) or opKind notin {attachedAsgn, attachedSink, attachedDup}: + # A hook reaching here has no effect list yet, i.e. it has not been + # effect-tracked. Propagating from its (still unset) type flags would + # spuriously mark the caller GC-unsafe/side-effecting: e.g. under + # `nim ic` a concrete `=destroy` reached through a generic + # instantiation is not analyzed before the instance body is tracked + # here. Skip all such hooks (generalizes #25940, which special-cased + # `=asgn`/`=sink`/`=dup`); once analyzed they carry an effect list and + # take the branch below. + let (isHook, _) = findHookKind(a.sym.name.s) + if not isHook: propagateEffects(tracked, n, a.sym) else: mergeRaises(tracked, effectList[exceptionEffects], n) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 9a230faf83..4c1a1b4cb3 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -294,13 +294,22 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0; expectedType: PT replaceTypeVarsS(cl, n.sym, result.typ) else: replaceTypeVarsS(cl, n.sym, replaceTypeVarsT(cl, n.sym.typ)) - if result.sym.kind == skField and result.sym.ast != nil and + if result.sym.kind == skField and (cl.owner == nil or result.sym.owner == cl.owner): - # instantiate default value of object/tuple field - var n = result.sym.ast - cl.c.fitDefaultNode(cl.c, n, result.sym.typ) - result.sym.ast = n - result.sym.typ = n.typ.skipIntLit(cl.c.idgen) + if result.sym.ast != nil: + # instantiate default value of object/tuple field + var n = result.sym.ast + cl.c.fitDefaultNode(cl.c, n, result.sym.typ) + result.sym.ast = n + result.sym.typ = n.typ.skipIntLit(cl.c.idgen) + elif result.typ != nil: + # The field SYM can be SHARED across the branches of an `nkRecWhen` (the + # generic body reuses one `value` PSym, so it carries the LAST branch's + # type), while the resolved field NODE carries the correct branch type. + # Sync the sym to the node so the instantiated field's sym-type and + # node-type agree (else a generic-object instance serializes a field + # whose sym-type diverges from its node-type -> loader/computeSize crash). + result.sym.typ = result.typ # sym type can be nil if was gensym created by macro, see #24048 if result.sym.typ != nil and result.sym.typ.kind == tyVoid: # don't add the 'void' field diff --git a/compiler/typekeys.nim b/compiler/typekeys.nim index d5e0b52e09..54092eb055 100644 --- a/compiler/typekeys.nim +++ b/compiler/typekeys.nim @@ -13,7 +13,7 @@ import std/[assertions, sets] import "../dist/nimony/src/lib" / [treemangler] -import "../dist/nimony/src/gear2" / modnames +import icmodnames import astdef, idents, options, lineinfos, msgs import ic / [enum2nif] diff --git a/compiler/types.nim b/compiler/types.nim index f4c7e74bc5..b6da7d4c12 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -633,6 +633,34 @@ proc lengthOrd*(conf: ConfigRef; t: PType): Int128 = let first = firstOrd(conf, t) result = last - first + One +const broadcastArrayThreshold* = 32 + ## `getNullValue` represents the default of an `array[N, T]` with `N` above this + ## as a single *broadcast* element — a one-son `nkBracket` standing for `N` + ## identical zero copies — instead of materialising `N` zero nodes. This keeps + ## huge zeroed arrays (e.g. SSZ byte buffers in nimbus) compact in the IC caches + ## (`.s.bif`/`.t.bif`), in the VM, and in the generated C (`{0}` zero-fills). + +proc isDefaultBroadcastArray*(n: PNode; conf: ConfigRef): bool = + ## True iff `n` is a broadcast default array: a single son standing for + ## `lengthOrd` identical zero copies. Identified by the explicit `nfBroadcast` + ## marker (set by `getNullValue`), NOT by `len == 1 < lengthOrd` — the latter + ## also matches an ordinary 1-element collection that happens to be an + ## `nkBracket` carrying an array type, e.g. a `@[a, b, c]` seq value shrunk to + ## length 1 by `setLen`/`delete` (its VM node keeps the array-literal type). + result = n != nil and n.kind == nkBracket and nfBroadcast in n.flags + +proc expandBroadcastArray*(n: PNode; conf: ConfigRef) = + ## Materialise a broadcast default array (see `isDefaultBroadcastArray`) into a + ## full `lengthOrd`-son `nkBracket`, each son a copy of the single default + ## element. Used by VM ops that index-address, mutate, or measure such a node; + ## the common read-only paths leave it compact. Clears `nfBroadcast` since the + ## node is now a fully materialised literal. + if isDefaultBroadcastArray(n, conf): + let total = toInt(lengthOrd(conf, n.typ.skipTypes(abstractInst))) + let elem = n[0] + for i in 1 ..< total: n.add copyTree(elem) + n.flags.excl nfBroadcast + # -------------- type equality ----------------------------------------------- type diff --git a/compiler/vm.nim b/compiler/vm.nim index 23c579dd17..0d9ab99b2e 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -702,6 +702,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = # A bodge, but this takes in `toOpenArray(rb, rc, rc)` and emits # nkTupleConstr(x, y, z) into the `regs[ra]`. These can later be used for calculating the slice we have taken. decodeBC(rkNode) + # Slicing/openArray needs the real length and per-element nodes, so a + # compact default array must be materialised first. + if isDefaultBroadcastArray(regs[ra].node, c.config): + expandBroadcastArray(regs[ra].node, c.config) let collection = regs[ra].node leftInd = regs[rb].intVal @@ -770,6 +774,15 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = regs[ra].node.intVal = src.strVal[idx].ord else: stackTrace(c, tos, pc, formatErrorIndexBound(idx, src.strVal.len-1)) + elif isDefaultBroadcastArray(src, c.config): + # `a[i]` on a compact default array yields `default(T)` directly, without + # ever materialising the (potentially huge) array — the point of the + # broadcast form. See `getNullValue`/`isDefaultBroadcastArray`. + let total = toInt(lengthOrd(c.config, src.typ.skipTypes(abstractInst))) + if idx <% total: + regs[ra].node = copyTree(src[0]) + else: + stackTrace(c, tos, pc, formatErrorIndexBound(idx, total-1)) elif src.kind notin {nkEmpty..nkFloat128Lit} and idx <% src.len: regs[ra].node = src[idx] else: @@ -781,6 +794,9 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = stackTrace(c, tos, pc, formatErrorIndexBound(regs[rc].intVal, high(int))) let idx = regs[rc].intVal.int let src = if regs[rb].kind == rkNode: regs[rb].node else: regs[rb].nodeAddr[] + # Taking the address of an element needs distinct, stable per-slot nodes, so + # a compact default array must be materialised first. + if isDefaultBroadcastArray(src, c.config): expandBroadcastArray(src, c.config) case src.kind of nkTupleConstr: let @@ -829,6 +845,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let idx = regs[rb].intVal.int assert regs[ra].kind == rkNode let arr = regs[ra].node + # Writing a slot materialises a compact default array into a full literal. + if isDefaultBroadcastArray(arr, c.config): expandBroadcastArray(arr, c.config) case arr.kind of nkTupleConstr: # refer to `opcSlice` let @@ -1031,6 +1049,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = case node.kind of nkTupleConstr: # refer to `of opcSlice` regs[ra].intVal = node[2].intVal - node[1].intVal + 1 - high + elif isDefaultBroadcastArray(node, c.config): + regs[ra].intVal = toInt(lengthOrd(c.config, node.typ.skipTypes(abstractInst))) - high else: # safeArrLen also return string node len # used when string is passed as openArray in VM diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 999b0756de..a560665b68 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1921,7 +1921,11 @@ proc genCheckedObjAccessAux(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags let strType = getSysType(c.graph, n.info, tyString) var msgReg: TDest = c.getTemp(strType) let fieldName = $accessExpr[1] - let msg = genFieldDefect(c.config, fieldName, disc.sym) + # Re-navigate the discriminant in the object type: under `nim ic` `disc.sym` is a + # field-use stub with a nil `owner`, which `genFieldDefect` dereferences. Look up the + # canonical discriminant field by name. Byte-neutral for non-IC (returns the same sym). + let dfield = lookupFieldAgain(accessExpr[0].typ, disc.sym) + let msg = genFieldDefect(c.config, fieldName, dfield) let strLit = newStrNode(msg, accessExpr[1].info) strLit.typ = strType c.genLit(strLit, msgReg) @@ -2025,8 +2029,20 @@ proc getNullValue(c: PCtx; typ: PType, info: TLineInfo; conf: ConfigRef): PNode getNullValueAux(c, t, t.n, result, conf, currPosition) of tyArray: result = newNodeIT(nkBracket, info, t) - for i in 0.. 0: result.add getNullValue(c, elemType(t), info, conf) + # For a large array, keep a single broadcast element (the default of every + # slot is identical) instead of `n` copies; `isDefaultBroadcastArray` + # consumers expand on demand. Small arrays stay fully materialised so the + # well-trodden paths are untouched. See `broadcastArrayThreshold`. + if n <= broadcastArrayThreshold: + for i in 1.. no importer re-sem) +# * an interface edit propagates to (`iface-edit`: an `*.iface.bif` +# importers cookie changes and >= 2 modules' +# `*.s.bif` codegen is rebuilt) +# +# File format (a `tests/ic/t*.nim` whose body, after the spec header, contains a +# line `#? metamorphic`): +# +# #? metamorphic +# #!FILE a.nim +# proc greet*(): string = "hi" +# #!FILE main.nim # `main.nim` is always the build root +# import a +# echo greet() +# #!STEP expect: hi +# #!FILE a.nim # re-emit a module to "edit" it +# proc greet*(): string = "hi" # identical content +# #!STEP expect: hi; noop +# +# `#!FILE ` blocks (re)write a module in the virtual file system; the +# accumulated file set is materialised before each `#!STEP`. A `#!STEP`'s +# attributes are `;`-separated, each either `key: value` or a bare flag: +# expect: noop body-edit iface-edit modules: clean +# The last step always also runs the clean==incremental check. + +type MetamorphicError = object of CatchableError + resultKind: TResultEnum + expected, given: string + +proc mmRaise(kind: TResultEnum, expected, given: string) = + var e = newException(MetamorphicError, given) + e.resultKind = kind + e.expected = expected + e.given = given + raise e + +proc isMetamorphicIcTest(content: string): bool = + for line in content.splitLines: + if line.strip == "#? metamorphic": return true + +proc snapshotDir(dir: string): Table[string, string] = + ## relative path -> raw file contents, for every file under `dir`. + result = initTable[string, string]() + if dirExists(dir): + for it in walkDirRec(dir): + result[it.relativePath(dir)] = readFile(it) + +proc changedPaths(prev, cur: Table[string, string]): seq[string] = + result = @[] + for k, v in cur: + if prev.getOrDefault(k) != v: result.add k + for k in prev.keys: + if k notin cur: result.add k + +proc isProvenance(path: string): bool = + ## Build-provenance sidecars that legitimately differ between a fresh build and + ## an edit-accumulated one (they record build history, not codegen). Excluded + ## only from the cross-build clean==incremental comparison — a *no-op* edit must + ## still leave even these untouched. + path.endsWith(".frontend.build.nif") + +proc stableBinary(path: string): string = + ## Contents of a linked executable past its header region, for comparing whether + ## two builds produced the same *code*. Linkers embed build-time-volatile fields + ## in the header (e.g. the mingw PE `TimeDateStamp` and its derived `CheckSum`), + ## so two builds seconds apart differ there even with identical codegen. Skipping + ## a generous fixed window keeps the clean-vs-incremental check about codegen. + const headerSkip = 4096 + var f: File + if not open(f, path, fmRead): + raise newException(IOError, "cannot open: " & path) + defer: close(f) + if getFileSize(f) > headerSkip: + setFilePos(f, headerSkip) + result = readAll(f) + +proc changedModuleCount(changed: seq[string]): int = + ## distinct modules whose codegen (`*.s.bif`) was rebuilt. + var mods: seq[string] = @[] + for p in changed: + if p.endsWith(".s.bif"): + let key = p.extractFilename.split('.')[0] + if key notin mods: mods.add key + result = mods.len + +proc runMetamorphicIcTest(r: var TResults; file: string; cat: Category; options: string) = + var test = TTest(cat: cat, name: file, options: options, + spec: initSpec(file), startTime: epochTime()) + test.spec.targets = {targetC} + inc r.total + + # Absolute paths: `nim ic` runs with `workingDir = buildDir`, so a relative + # `--nimcache` would resolve against the build dir, not where we read it back. + let buildDir = (file.changeFileExt("") & "_mm").absolutePath + let nc = buildDir / "nc" + let bin = buildDir / "prog".addFileExt(ExeExt) + removeDir(buildDir) + createDir(buildDir) + + template compileIc(): untyped = + execCmdEx2(compilerPrefix, ["ic", "--hint:Conf:off", "--warnings:off", + "--nimcache:" & nc, "--out:" & bin, "main.nim"], + workingDir = buildDir) + + # Parse the source into a flat op list: ("file", name, content) | ("step", attrs, ""). + type OpKind = enum opFile, opStep + type Op = object + kind: OpKind + a, b: string + var ops: seq[Op] = @[] + block parse: + var curName = "" + var buf = "" + template flushFile() = + if curName.len > 0: ops.add Op(kind: opFile, a: curName, b: buf) + curName = ""; buf = "" + for raw in readFile(file).splitLines: + let s = raw.strip + if s.startsWith("#!FILE"): + flushFile() + curName = s["#!FILE".len .. ^1].strip + elif s.startsWith("#!STEP"): + flushFile() + ops.add Op(kind: opStep, a: s["#!STEP".len .. ^1].strip) + elif curName.len > 0: + buf.add raw; buf.add "\n" + let lastStep = block: + var n = 0 + for o in ops: + if o.kind == opStep: inc n + n + + var vfs = initTable[string, string]() + var prevSnap = initTable[string, string]() + var prevBin = "" + var stepIdx = 0 + try: + for o in ops: + if o.kind == opFile: + vfs[o.a] = o.b + continue + inc stepIdx + let where = "step " & $stepIdx + # Parse step attributes. + var attrs = initTable[string, string]() + for part in o.a.split(';'): + let p = part.strip + if p.len == 0: continue + let c = p.find(':') + if c >= 0: attrs[p[0 ..< c].strip] = p[c+1 .. ^1].strip + else: attrs[p] = "" + + for fn, content in vfs: writeFile(buildDir / fn, content) + let (_, cout, ccode) = compileIc() + if ccode != 0: + mmRaise(reBuildFailed, "", where & ": `nim ic` failed:\n" & cout) + let (_, rout, rcode) = execCmdEx2(bin.absolutePath, [], workingDir = buildDir) + if rcode != 0: + mmRaise(reBuildFailed, "", where & ": program exited with " & $rcode & ":\n" & rout) + if "expect" in attrs: + let want = attrs["expect"].replace("\\n", "\n") + if rout.strip == want.strip: discard + else: mmRaise(reOutputsDiffer, want, where & " output:\n" & rout.strip) + + let snap = snapshotDir(nc) + let binBytes = stableBinary(bin) + if stepIdx > 1: + let changed = changedPaths(prevSnap, snap) + if "noop" in attrs and (changed.len != 0 or binBytes != prevBin): + mmRaise(reOutputsDiffer, "no artifact change", + where & ": no-op edit changed " & $changed.len & " cache file(s): " & changed.join(", ")) + if "body-edit" in attrs: + for p in changed: + if p.endsWith(".iface.bif"): + mmRaise(reOutputsDiffer, "no interface change", + where & ": body-only edit changed an interface cookie: " & p) + if "iface-edit" in attrs: + var sawIface = false + for p in changed: + if p.endsWith(".iface.bif"): sawIface = true + if not sawIface: + mmRaise(reOutputsDiffer, "interface change", where & ": interface edit changed no `*.iface.bif` cookie") + if changedModuleCount(changed) < 2: + mmRaise(reOutputsDiffer, "propagation to importer", + where & ": interface edit did not propagate (only " & $changedModuleCount(changed) & " module rebuilt)") + if "modules" in attrs: + let want = parseInt(attrs["modules"]) + let got = changedModuleCount(changed) + if got != want: + mmRaise(reOutputsDiffer, $want & " modules rebuilt", where & ": " & $got & " module(s) rebuilt") + prevSnap = snap + prevBin = binBytes + + if "clean" in attrs or stepIdx == lastStep: + removeDir(nc) + let (_, cout2, ccode2) = compileIc() + if ccode2 != 0: + mmRaise(reBuildFailed, "", where & ": clean rebuild failed:\n" & cout2) + let cleanSnap = snapshotDir(nc) + let cleanBin = stableBinary(bin) + if cleanBin != binBytes: + mmRaise(reOutputsDiffer, "clean binary == incremental binary", + where & ": clean rebuild produced a different binary") + var diff: seq[string] = @[] + for p in changedPaths(snap, cleanSnap): + if not isProvenance(p): diff.add p + if diff.len != 0: + mmRaise(reOutputsDiffer, "clean cache == incremental cache", + where & ": clean rebuild differs in " & $diff.len & " cache file(s): " & diff.join(", ")) + prevSnap = cleanSnap + prevBin = cleanBin + finishTest(r, test, targetC, "", "", "", reSuccess) + inc r.passed + except MetamorphicError: + let e = (ref MetamorphicError)(getCurrentException()) + finishTest(r, test, targetC, "", e.expected, e.given, e.resultKind) + proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; isNavigatorTest: bool) = template editedTest() = @@ -498,11 +728,18 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; const tempExt = "_temp.nim" for it in walkDirRec(testsDir): + # `_mm` directories hold materialised modules + nimcache for metamorphic + # tests; never collect their files as tests in their own right. + if "_mm" in it: continue if isTestFile(it) and not it.endsWith(tempExt): + let content = readFile(it) + if isMetamorphicIcTest(content): + runMetamorphicIcTest(r, it, cat, options) + continue + let nimcache = nimcacheDir(it, options, targetC) removeDir(nimcache) - let content = readFile(it) for fragment in content.split("#!EDIT!#"): let file = it.replace(".nim", tempExt) writeFile(file, fragment) diff --git a/tests/ic/mdefconverter.nim b/tests/ic/mdefconverter.nim new file mode 100644 index 0000000000..d0a23f801b --- /dev/null +++ b/tests/ic/mdefconverter.nim @@ -0,0 +1,2 @@ + +converter toBool*(x: int): bool = x != 0 diff --git a/tests/ic/memit.nim b/tests/ic/memit.nim new file mode 100644 index 0000000000..5b02334424 --- /dev/null +++ b/tests/ic/memit.nim @@ -0,0 +1,14 @@ +# Helper for temit.nim: a NON-main module with a module-scope `{.emit.}` that +# introduces a C macro consumed by an `{.importc, nodecl.}` const. The IC backend +# reload used to drop top-level emit pragmas — writeToplevelNode wrote them to the +# by-symbol-index implementation section, where a symbol-less pragma is never +# reloaded — so the generated C lost the `#define` and failed to compile with +# "use of undeclared identifier". Mirrors lib/pure/concurrency/cpuinfo.nim's +# `#include ` + `CTL_HW`/`HW_NCPU` importc pattern. +{.emit: """/*TYPESECTION*/ +#define NIM_IC_EMIT_ANSWER 42 +""".} + +let icEmitAnswer {.importc: "NIM_IC_EMIT_ANSWER", nodecl.}: cint + +proc emitAnswer*(): int = int(icEmitAnswer) diff --git a/tests/ic/memptycaller.nim b/tests/ic/memptycaller.nim new file mode 100644 index 0000000000..88c8eeefb1 --- /dev/null +++ b/tests/ic/memptycaller.nim @@ -0,0 +1,9 @@ +# Calls `emptyOwned` from a DIFFERENT module than the one that owns it, so the +# reference at link time must resolve to a definition the owning module emits. + +import memptyowned + +proc callEmpty*(cond: bool) = + if cond: + emptyOwned() + echo "called ", cond diff --git a/tests/ic/memptyowned.nim b/tests/ic/memptyowned.nim new file mode 100644 index 0000000000..a1938400ee --- /dev/null +++ b/tests/ic/memptyowned.nim @@ -0,0 +1,12 @@ +# Helper for `temptyowned`: exports a concrete proc whose body folds to +# nothing (an `nkEmpty` body, like Nimbus' `extras.incInternalErrors` when the +# metrics counter's `.inc()` expands to a no-op under `-u:metrics`). The proc is +# NOT called within its own module — only `memptycaller` references it — so the +# per-module backend's owned-routine seeding is the ONLY thing that can emit it. + +template maybe*(x: untyped) = + when false: + x + +proc emptyOwned*() = + maybe(echo "unreachable") diff --git a/tests/ic/minitordera.nim b/tests/ic/minitordera.nim new file mode 100644 index 0000000000..a35ac0eed7 --- /dev/null +++ b/tests/ic/minitordera.nim @@ -0,0 +1,29 @@ +# Helper for tinitorder (not a test itself; no `discard`). +# +# Imports minitorderb and, in its OWN init, reads the state minitorderb set up. +# With the wrong (importer-first) init order `gBState` is still 0 here. It also +# allocates a seq in its init so that under `--mm:refc` the GC (set up by the +# system module's init) must already be live — i.e. the system module's init has +# to be ordered first. + +import minitorderb + +var + gASawB = -1 + gAItems: seq[int] + +proc getASawB*(): int = gASawB +proc getACount*(): int = gAItems.len + +proc recordA() = + gASawB = getBState() + gAItems = @[1, 2, 3] + # Allocate (and drop) enough garbage to force a GC cycle DURING module init. + # Under refc that runs a conservative stack scan, which needs the main + # thread's stack bottom already set — i.e. `initStackBottomWith` must run + # before the module inits, not after them. + for i in 0 ..< 100_000: + let s = @[i, i + 1, i + 2] + doAssert s.len == 3 + +recordA() diff --git a/tests/ic/minitorderb.nim b/tests/ic/minitorderb.nim new file mode 100644 index 0000000000..8c406e8e94 --- /dev/null +++ b/tests/ic/minitorderb.nim @@ -0,0 +1,15 @@ +# Helper for tinitorder (not a test itself; no `discard`). +# +# An imported module whose INIT sets module-level state at runtime. Under the +# per-module backend its init must run BEFORE any importer's init (the imported +# module is a dependency → post-order). With the buggy importer-first order this +# module's `setupB` runs too late and importers observe `gBState == 0`. + +var gBState: int + +proc getBState*(): int = gBState + +proc setupB() = + gBState = 42 + +setupB() diff --git a/tests/ic/mmethanimal.nim b/tests/ic/mmethanimal.nim new file mode 100644 index 0000000000..76e0bcaf68 --- /dev/null +++ b/tests/ic/mmethanimal.nim @@ -0,0 +1,15 @@ +# Helper for tmethitanium: the OWNER module of a `{.base.}` method. Its concrete +# base body is emitted here; the whole-program dispatcher is synthesized into the +# main module. Under `--debugger:native` the backend uses the Itanium mangling +# scheme, which encodes the signature instead of the `disamb`, so the base method +# and its same-signature dispatcher want the identical clean C name. The +# clean-vs-unique tie-break used to depend on a per-MODULE set (`mangledPrcs`), +# which the per-module IC backend cannot share — the base mangled clean at this +# owner but `speak_u` (an unstable `itemId.item`) at every demander, so it was +# defined once and referenced under names nobody defined. See +# ccgutils.makeUnique (disamb, not itemId) + ccgtypes.fillBackendName. + +type Animal* = ref object of RootObj + +method speak*(a: Animal): string {.base.} = + "generic-animal-sound" diff --git a/tests/ic/mmethdog.nim b/tests/ic/mmethdog.nim new file mode 100644 index 0000000000..22d0de7130 --- /dev/null +++ b/tests/ic/mmethdog.nim @@ -0,0 +1,14 @@ +# Helper for tmethitanium: an override in a DIFFERENT module than the base, plus +# a `procCall` super-reference to the base from this non-owner module. That +# cross-module reference to the base impl is what diverged from the base's +# definition name under the per-module Itanium mangling. + +import mmethanimal + +type Dog* = ref object of Animal + +method speak*(a: Dog): string = + "woof" + +proc speakBoth*(a: Dog): string = + procCall(speak(Animal(a))) & "/" & speak(a) diff --git a/tests/ic/temit.nim b/tests/ic/temit.nim new file mode 100644 index 0000000000..214ab2f90b --- /dev/null +++ b/tests/ic/temit.nim @@ -0,0 +1,7 @@ +# Regression: a module-scope `{.emit.}` in an imported module must survive the +# `nim ic` backend reload (see memit.nim). Before the fix the dropped `#define` +# made the generated C fail to compile, so `nim ic` exited non-zero. +import memit + +doAssert emitAnswer() == 42 +echo emitAnswer() diff --git a/tests/ic/temptyowned.nim b/tests/ic/temptyowned.nim new file mode 100644 index 0000000000..2d37ebe034 --- /dev/null +++ b/tests/ic/temptyowned.nim @@ -0,0 +1,18 @@ +discard """ +output: '''called false +done''' +""" + +# Regression test for the per-module IC backend dropping an owned routine whose +# body folds to `nkEmpty`. `memptyowned.emptyOwned` is a real, concrete, owned +# proc with an empty body; `memptycaller.callEmpty` references it across a module +# boundary. The owning module's `ownsRuntimeRoutine` seeding used to reject any +# routine with an `nkEmpty` body, so nobody emitted `emptyOwned` -> undefined +# reference at link (mirrors Nimbus `ncli` linking `extras.incInternalErrors`). +# Whole-program cgen always emits it as `void emptyOwned(void){}`; the per-module +# backend must too. + +import memptycaller + +callEmpty(false) +echo "done" diff --git a/tests/ic/tinitorder.nim b/tests/ic/tinitorder.nim new file mode 100644 index 0000000000..93d0d94f4c --- /dev/null +++ b/tests/ic/tinitorder.nim @@ -0,0 +1,23 @@ +discard """ +output: '''42 3''' +""" + +# Regression test for per-module-backend module-init ORDERING. +# +# NimMain must call each module's init in DEPENDENCY (post-order) order: an +# imported module's init has to run before its importer's. The whole-program +# backend gets this for free (it iterates `modulesClosed`, built in module-finish +# order); the per-module backend reconstructs it in `nifbackend`. The earlier +# code iterated `bl.mods` by POSITION, which runs importers before their +# dependencies (an importer gets a lower file position than the modules it +# imports) — and the system module (which runs `initGC` in its init) was not +# ordered first at all. +# +# Module chain: tinitorder -> minitordera -> minitorderb. `minitorderb`'s init +# sets a global to 42; `minitordera`'s init reads it (and allocates a seq). With +# the buggy order `minitordera` runs first and reads 0 (or, under refc, crashes +# allocating before the GC is up). Correct order prints `42 3`. + +import minitordera + +echo getASawB(), " ", getACount() diff --git a/tests/ic/tmeta_async.nim b/tests/ic/tmeta_async.nim new file mode 100644 index 0000000000..51e64ac24d --- /dev/null +++ b/tests/ic/tmeta_async.nim @@ -0,0 +1,46 @@ +discard """ + description: '''metamorphic IC: async/await across an edit (continuations + clean==incremental)''' +""" + +# Async is the most cache-fragile area under `nim ic`: the `{.async.}` transform +# generates continuation closures and lifts environments, and those lowered +# bodies must survive incremental edits and converge to exactly what a clean +# build produces. This drives an edit to an async proc body and asserts the +# importer is NOT rebuilt (the lifted continuation stays local to its module), +# a no-op changes nothing, and the final state is byte-identical to a clean +# build. See doc/ic_ideas.md and [[ic-nimbus-test]] (async was the long pole). + +#? metamorphic + +#!FILE worker.nim +import std/asyncdispatch +proc compute*(x: int): Future[int] {.async.} = + await sleepAsync(0) + result = x * 2 + +#!FILE main.nim +import std/asyncdispatch, worker +echo waitFor compute(21) + +#!STEP expect: 42 + +# --- edit the async proc body. The continuation env lives in `worker`, so only +# `worker` rebuilds; `main` (the caller) is left untouched -> modules: 1. +# (An async body edit does perturb `worker`'s interface cookie via the +# generated env type, so this is not asserted as a pure `body-edit`.) +#!FILE worker.nim +import std/asyncdispatch +proc compute*(x: int): Future[int] {.async.} = + await sleepAsync(0) + result = x * 3 + +#!STEP expect: 63; modules: 1 + +# --- re-emit identical content: nothing may change, lifted closures included. +#!FILE worker.nim +import std/asyncdispatch +proc compute*(x: int): Future[int] {.async.} = + await sleepAsync(0) + result = x * 3 + +#!STEP expect: 63; noop diff --git a/tests/ic/tmeta_exprcase.nim b/tests/ic/tmeta_exprcase.nim new file mode 100644 index 0000000000..ea6fdcdb13 --- /dev/null +++ b/tests/ic/tmeta_exprcase.nim @@ -0,0 +1,64 @@ +discard """ + description: '''metamorphic IC: expression-based `let x = case ...` idiom''' +""" + +# nimbus-eth2 leans on expression-style code (`let x = case ...` rather than +# statement assignment). This exercises that construct through the incremental +# path: a body edit that adds a `case` branch stays local to the module, while a +# return-type change (interface edit) propagates to the importer even though the +# importer's source is byte-identical. See doc/ic_ideas.md. + +#? metamorphic + +#!FILE classify.nim +proc classify*(n: int): string = + let kind = case n + of 0: "zero" + of 1, 2, 3: "small" + else: "big" + result = kind & "(" & $n & ")" + +#!FILE main.nim +import classify +echo classify(0), " ", classify(2), " ", classify(99) + +#!STEP expect: zero(0) small(2) big(99) + +# --- body edit: add a `case` branch and tweak a label. The signature is +# unchanged, so no interface cookie changes and only `classify` rebuilds. +#!FILE classify.nim +proc classify*(n: int): string = + let kind = case n + of 0: "ZERO" + of 1, 2, 3: "small" + of 4, 5, 6: "medium" + else: "big" + result = kind & "(" & $n & ")" + +#!STEP expect: ZERO(0) small(2) big(99); body-edit; modules: 1 + +# --- re-emit identical content: nothing may change. +#!FILE classify.nim +proc classify*(n: int): string = + let kind = case n + of 0: "ZERO" + of 1, 2, 3: "small" + of 4, 5, 6: "medium" + else: "big" + result = kind & "(" & $n & ")" + +#!STEP expect: ZERO(0) small(2) big(99); noop + +# --- interface edit: the expression-`case` now yields `int`, changing +# `classify`'s return type. `main`'s source is byte-identical (`echo` prints +# either) yet must re-sem & recodegen -> the cookie changes and 2 modules +# rebuild. The final step also runs the clean==incremental check. +#!FILE classify.nim +proc classify*(n: int): int = + result = case n + of 0: 0 + of 1, 2, 3: 1 + of 4, 5, 6: 5 + else: 9 + +#!STEP expect: 0 1 9; iface-edit; modules: 2 diff --git a/tests/ic/tmeta_generic.nim b/tests/ic/tmeta_generic.nim new file mode 100644 index 0000000000..35f26f2e61 --- /dev/null +++ b/tests/ic/tmeta_generic.nim @@ -0,0 +1,33 @@ +discard """ + description: '''metamorphic IC: generic instantiation cache stability across edits''' +""" + +# A generic lives in `gen` but is *instantiated* in `main` (the per-module +# backend emits instance bodies at the instantiation site). Editing the generic +# body must therefore rebuild both `gen` and `main`, and an incremental edit must +# still converge to exactly the same artifacts as a clean build. This exercises +# the static/generic-instance cache path that has historically been bug-prone. + +#? metamorphic + +#!FILE gen.nim +proc box*[T](x: T): seq[T] = @[x, x] + +#!FILE main.nim +import gen +echo box(3).len, " ", box("hi")[0] + +#!STEP expect: 2 hi + +# --- edit the generic body: the importer holds the instantiations, so both the +# definer and the instantiation site rebuild (2 modules). +#!FILE gen.nim +proc box*[T](x: T): seq[T] = @[x, x, x] + +#!STEP expect: 3 hi; modules: 2 + +# --- re-emit identical content: nothing may change. +#!FILE gen.nim +proc box*[T](x: T): seq[T] = @[x, x, x] + +#!STEP expect: 3 hi; noop diff --git a/tests/ic/tmeta_ortype.nim b/tests/ic/tmeta_ortype.nim new file mode 100644 index 0000000000..132d45356e --- /dev/null +++ b/tests/ic/tmeta_ortype.nim @@ -0,0 +1,61 @@ +discard """ + description: '''metamorphic IC: or-type (type-class union) idiom from nimbus forks.nim''' +""" + +# nimbus-eth2 writes a lot of generic code over `A | B | C` type-class unions +# (consensus forks). This models that idiom: a `Fruit = Apple | Banana` union +# with a generic `describe[T: Fruit]` dispatched by `when T is ...`. The generic +# is instantiated in `main`, so editing its body rebuilds both modules, and the +# incremental result must match a clean build. See doc/ic_ideas.md. + +#? metamorphic + +#!FILE forks.nim +type + Apple* = object + weight*: int + Banana* = object + length*: int + Fruit* = Apple | Banana + +proc describe*[T: Fruit](x: T): string = + when T is Apple: "apple " & $x.weight + else: "banana " & $x.length + +#!FILE main.nim +import forks +echo describe(Apple(weight: 5)), " | ", describe(Banana(length: 9)) + +#!STEP expect: apple 5 | banana 9 + +# --- edit the generic body (no signature change). The union/constraint is +# untouched, so no interface cookie changes; but `main` holds the two +# instantiations, so both modules' codegen rebuilds. +#!FILE forks.nim +type + Apple* = object + weight*: int + Banana* = object + length*: int + Fruit* = Apple | Banana + +proc describe*[T: Fruit](x: T): string = + when T is Apple: "APPLE " & $x.weight + else: "BANANA " & $x.length + +#!STEP expect: APPLE 5 | BANANA 9; body-edit; modules: 2 + +# --- re-emit identical content: nothing may change. +#!FILE forks.nim +type + Apple* = object + weight*: int + Banana* = object + length*: int + Fruit* = Apple | Banana + +proc describe*[T: Fruit](x: T): string = + when T is Apple: "APPLE " & $x.weight + else: "BANANA " & $x.length + +#!STEP expect: APPLE 5 | BANANA 9; noop diff --git a/tests/ic/tmeta_result.nim b/tests/ic/tmeta_result.nim new file mode 100644 index 0000000000..5ed596ea54 --- /dev/null +++ b/tests/ic/tmeta_result.nim @@ -0,0 +1,93 @@ +discard """ + description: '''metamorphic IC: Result[T, E] (nim-results style) variant-object idiom''' +""" + +# nimbus-eth2 threads errors through nim-results' `Result[T, E]`, a generic +# *variant* (case) object. This models a minimal hermetic version and exercises +# the generic-variant cache path across edits: a body edit of a generic accessor, +# adding a public overload (an interface edit), and a no-op — with a final +# clean==incremental check. See doc/ic_ideas.md. + +#? metamorphic + +#!FILE results.nim +type + ResultKind = enum rOk, rErr + Result*[T, E] = object + case kind: ResultKind + of rOk: v: T + of rErr: e: E + +proc ok*[T, E](x: T): Result[T, E] = Result[T, E](kind: rOk, v: x) +proc err*[T, E](x: E): Result[T, E] = Result[T, E](kind: rErr, e: x) +proc isOk*[T, E](r: Result[T, E]): bool = r.kind == rOk +proc get*[T, E](r: Result[T, E]): T = r.v +proc error*[T, E](r: Result[T, E]): E = r.e + +#!FILE main.nim +import results +proc parse(s: string): Result[int, string] = + if s == "42": ok[int, string](42) + else: err[int, string]("bad: " & s) +let a = parse("42") +let b = parse("x") +echo (if a.isOk: $a.get else: a.error), " ", (if b.isOk: $b.get else: b.error) + +#!STEP expect: 42 bad: x + +# --- body-only edit of a generic accessor (`error`): signature unchanged, so no +# interface cookie changes; the importer holds the instantiation, so both +# modules' codegen rebuilds. +#!FILE results.nim +type + ResultKind = enum rOk, rErr + Result*[T, E] = object + case kind: ResultKind + of rOk: v: T + of rErr: e: E + +proc ok*[T, E](x: T): Result[T, E] = Result[T, E](kind: rOk, v: x) +proc err*[T, E](x: E): Result[T, E] = Result[T, E](kind: rErr, e: x) +proc isOk*[T, E](r: Result[T, E]): bool = r.kind == rOk +proc get*[T, E](r: Result[T, E]): T = r.v +proc error*[T, E](r: Result[T, E]): E = "ERR:" & r.e + +#!STEP expect: 42 ERR:bad: x; body-edit; modules: 2 + +# --- re-emit identical content: nothing may change. +#!FILE results.nim +type + ResultKind = enum rOk, rErr + Result*[T, E] = object + case kind: ResultKind + of rOk: v: T + of rErr: e: E + +proc ok*[T, E](x: T): Result[T, E] = Result[T, E](kind: rOk, v: x) +proc err*[T, E](x: E): Result[T, E] = Result[T, E](kind: rErr, e: x) +proc isOk*[T, E](r: Result[T, E]): bool = r.kind == rOk +proc get*[T, E](r: Result[T, E]): T = r.v +proc error*[T, E](r: Result[T, E]): E = "ERR:" & r.e + +#!STEP expect: 42 ERR:bad: x; noop + +# --- interface edit: add a public `get` overload (a new exported signature). +# `main` doesn't call it, yet importing `results` whose interface changed +# forces a re-sem; the cookie changes and >= 2 modules rebuild. The final +# step also runs the clean==incremental check. +#!FILE results.nim +type + ResultKind = enum rOk, rErr + Result*[T, E] = object + case kind: ResultKind + of rOk: v: T + of rErr: e: E + +proc ok*[T, E](x: T): Result[T, E] = Result[T, E](kind: rOk, v: x) +proc err*[T, E](x: E): Result[T, E] = Result[T, E](kind: rErr, e: x) +proc isOk*[T, E](r: Result[T, E]): bool = r.kind == rOk +proc get*[T, E](r: Result[T, E], fallback: T): T = (if r.kind == rOk: r.v else: fallback) +proc get*[T, E](r: Result[T, E]): T = r.v +proc error*[T, E](r: Result[T, E]): E = "ERR:" & r.e + +#!STEP expect: 42 ERR:bad: x; iface-edit; modules: 2 diff --git a/tests/ic/tmeta_smoke.nim b/tests/ic/tmeta_smoke.nim new file mode 100644 index 0000000000..d8f057dab3 --- /dev/null +++ b/tests/ic/tmeta_smoke.nim @@ -0,0 +1,50 @@ +discard """ + description: '''metamorphic IC: clean==incremental, no-op stability, body vs interface boundary''' +""" + +# This is a *metamorphic* IC test (see the `#? metamorphic` marker and the +# runner in testament/categories.nim). It drives a sequence of cross-module +# edits through `nim ic` in one fixed build directory and checks the invariants +# the incremental backend must uphold (doc/ic_ideas.md). + +#? metamorphic + +#!FILE a.nim +proc greet*(): string = "hi" +proc secret(): int = 41 # private, body-only churn target +proc value*(): int = secret() + 1 + +#!FILE main.nim +import a +echo greet(), " ", value() + +#!STEP expect: hi 42 + +# --- body-only edit: a private body changes, no signature does. +# => no `*.iface.bif` cookie changes, exactly 1 module's codegen rebuilds, +# the importer is left untouched. +#!FILE a.nim +proc greet*(): string = "hi" +proc secret(): int = 999 +proc value*(): int = secret() + 1 + +#!STEP expect: hi 1000; body-edit; modules: 1 + +# --- no-op edit: re-emit byte-identical content. Nothing downstream may change. +#!FILE a.nim +proc greet*(): string = "hi" +proc secret(): int = 999 +proc value*(): int = secret() + 1 + +#!STEP expect: hi 1000; noop + +# --- interface edit: `value`'s return type changes (a signature change) while +# main.nim's source stays byte-identical. The interface cookie must change +# and the importer must be re-sem'd & recodegen'd (>= 2 modules rebuilt). +# The final step also runs the clean==incremental check. +#!FILE a.nim +proc greet*(): string = "hi" +proc secret(): int = 999 +proc value*(): int64 = secret().int64 + 1 + +#!STEP expect: hi 1000; iface-edit diff --git a/tests/ic/tmeta_transitive.nim b/tests/ic/tmeta_transitive.nim new file mode 100644 index 0000000000..aec23f7d66 --- /dev/null +++ b/tests/ic/tmeta_transitive.nim @@ -0,0 +1,39 @@ +discard """ + description: '''metamorphic IC: edit propagation across a 3-module import chain''' +""" + +# Chain: main -> b -> a. Demonstrates that a body edit stays local to the edited +# module, while an interface edit propagates to its direct importer but stops +# where the next signature is unchanged. See doc/ic_ideas.md and the runner in +# testament/categories.nim. + +#? metamorphic + +#!FILE a.nim +proc base*(): int = 1 + +#!FILE b.nim +import a +proc mid*(): int = base() + 10 + +#!FILE main.nim +import b +echo mid() + +#!STEP expect: 11 + +# --- body-only edit of `a.base`: no signature changes, so nothing re-sems; +# only module `a`'s own codegen rebuilds. +#!FILE a.nim +proc base*(): int = 7 + +#!STEP expect: 17; body-edit; modules: 1 + +# --- interface edit of `a.base` (return type int -> int64). `b` uses `base`, so +# `a`'s cookie change forces `b` to re-sem & recodegen; but `b.mid`'s own +# signature is unchanged, so `main` is NOT rebuilt -> exactly 2 modules. +# The final step also runs the clean==incremental check. +#!FILE a.nim +proc base*(): int64 = 7 + +#!STEP expect: 17; iface-edit; modules: 2 diff --git a/tests/ic/tmethitanium.nim b/tests/ic/tmethitanium.nim new file mode 100644 index 0000000000..f226af37c1 --- /dev/null +++ b/tests/ic/tmethitanium.nim @@ -0,0 +1,33 @@ +discard """ + output: '''woof +generic-animal-sound +generic-animal-sound/woof''' +""" + +# NOTE: the `--debugger:native` that triggers the Itanium mangling lives in the +# sibling `tmethitanium_temp.nim.cfg` (the IC test harness compiles the generated +# `_temp.nim` and does not thread a `matrix`/`$options` switch into the cg +# children; a project cfg is read by the driver, which forwards it). + +# Regression test: under `nim ic --debugger:native` the per-module backend uses +# the Itanium C name mangling, which encodes the signature and drops the +# `disamb`. A `{.base.}` method (owner module mmethanimal) and its whole-program +# dispatcher (synthesized into this main module) then share a signature, so the +# clean-name uniqueness probe (`m.g.mangledPrcs`) — which only sees the current +# module — gave the base a clean name at its owner but an unstable +# `itemId.item`-based `speak_u` at each demander. Result: the base was defined +# once (clean) but referenced under names defined nowhere ("undefined reference +# to speak_u1") while the dispatcher collided with the clean base ("multiple +# definition of speak"). This was the bulk of nimbus-eth2's libp2p method link +# failures under `nim ic`. Fixed by making the Itanium scheme use the stable +# `disamb` (ccgutils.makeUnique) and always uniquify routine names under the +# per-module backend (ccgtypes.fillBackendName), plus forwarding +# `--debugger:native` to the cg children (deps.computeForwardedArgs). + +import mmethanimal, mmethdog + +let a: Animal = Dog() +echo speak(a) # dispatches -> override +let b: Animal = Animal() +echo speak(b) # dispatches -> base body +echo speakBoth(Dog()) # procCall to base from non-owner module diff --git a/tests/ic/tstaticgenfield.nim b/tests/ic/tstaticgenfield.nim new file mode 100644 index 0000000000..690245529e --- /dev/null +++ b/tests/ic/tstaticgenfield.nim @@ -0,0 +1,38 @@ +discard """ + output: '''9''' +""" + +# Regression test for object-field serialization of static-generic instances +# under `nim ic`. +# +# A generic object's instances SHARE one field PSym (same itemId) while each +# instance carries a DISTINCT field type, e.g. `Digest[32].data: array[32,byte]` +# vs `Digest[48].data: array[48,byte]` (this is exactly nimcrypto's `MDigest`, +# which crashed compiling nimbus's `altair.nim`). The `.s.bif` writer DEFs each +# field once inside its owning type's reclist and references it as a bare SymUse +# elsewhere, deduping by a per-Writer `emittedFieldSyms` set. That set wrongly +# spanned DIFFERENT type reclists: after the first instance's `data` def, every +# other instance's reclist got a typeless SymUse stub instead of its own typed +# def. On load that field had a nil `typ`/`owner`, and `=destroy` lifting +# (`liftdestructors.fillBodyObj`) dereferenced it -> SIGSEGV. The fix scopes the +# dedup per-reclist so each instance reclist is a self-contained typed def. +# +# The `seq` field forces `=destroy` to be lifted for `Outer`, which walks the +# reclists of both `Digest` instances (the crash path). + +type + Digest[n: static int] = object + data: array[n, byte] + Outer = object + a: Digest[32] + b: Digest[48] + s: seq[int] + +proc use(o: Outer): int = + result = o.a.data[0].int + o.b.data[0].int + o.s.len + +var o: Outer +o.a.data[0] = 4 +o.b.data[0] = 2 +o.s = @[1, 2, 3] +echo use(o) diff --git a/tests/vm/tsetlen.nim b/tests/vm/tsetlen.nim index 9fd30f3319..dc18baf422 100644 --- a/tests/vm/tsetlen.nim +++ b/tests/vm/tsetlen.nim @@ -28,3 +28,17 @@ block: fun[(int, string)]() fun[ref Foo]() fun[seq[int]]() + +block: # shrinking an `@[...]` seq literal in the VM + # A `@[a, b, c]` seq value keeps the array-literal type in the VM; shrinking it + # to length 1 must not be misread as a broadcast default array (was: the whole + # thing collapsed to `len` copies of the first element). + proc shrink = + var s = @[10, 20, 30] + s.setLen(1) + doAssert s == @[10] + var t = @["foo", "bar"] + t.delete(1) + doAssert t == @["foo"] + static: shrink() + shrink()