diff --git a/compiler/ast.nim b/compiler/ast.nim index b04a102b20..93688df604 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -36,6 +36,13 @@ proc setupProgram*(config: ConfigRef; cache: IdentCache) = when not defined(nimKochBootstrap): program = createDecodeContext(config, cache) +proc setIcMainModule*(fileIdx: FileIndex) = + ## Tells the IC loader which module is being compiled fresh, so that + ## re-exports of that module's symbols by dependencies are not loaded as + ## duplicate stubs. + when not defined(nimKochBootstrap): + ast2nif.setMainModule(program, fileIdx) + template loadSym(s: PSym) = ## Loads a symbol from NIF file if it's in Partial state. when not defined(nimKochBootstrap): diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 8847af32b7..841a982d73 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -161,14 +161,18 @@ type #writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later writtenPackages: HashSet[string] -const - # Symbol kinds that are always local to a proc and should never have module suffix - skLocalSymKinds = {skParam, skForVar, skResult, skTemp} - proc isLocalSym(sym: PSym): bool {.inline.} = - sym.kindImpl in skLocalSymKinds or - (sym.kindImpl in {skVar, skLet} and {sfGlobal, sfThread} * sym.flagsImpl == {} and - (sym.ownerFieldImpl == nil or sym.ownerFieldImpl.kindImpl != skModule)) + ## Params (and generic params, see `writeSymDef`) are emitted as *global* + ## (module-suffixed) names so their `sdef` gets an index entry and is + ## resolvable by index lookup even when referenced from a different index entry + ## than the one that physically contains the definition: generic params of a + ## forward declaration vs its implementation, and proc-type params shared + ## between an enclosing proc and a nested object's proc-type field. The + ## per-module `disamb` counter keeps `name.disamb.module` unique, so this is + ## clash-free. The kinds below are only ever used within the single index entry + ## of their owning routine, so they stay local (smaller index, and the codegen + ## name mangler only handles real module owners). + false proc toNifSymName(w: var Writer; sym: PSym): string = ## Generate NIF name for a symbol: local names are `ident.disamb`, @@ -301,24 +305,6 @@ proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) = dest.addStrLit lib.name writeNode w, dest, lib.path -proc collectGenericParams(w: var Writer; n: PNode) = - ## Pre-collect generic param symbols into w.locals before writing the type. - ## This ensures generic params get consistent short names, and their sdefs - ## are written in the type (where lazy loading can find them). - if n == nil: return - case n.kind - of nkSym: - if n.sym != nil and w.inProc > 0: - w.locals.incl(n.sym.itemId) - of nkIdentDefs, nkVarTuple: - for i in 0 ..< max(0, n.len - 2): - collectGenericParams(w, n[i]) - of nkGenericParams: - for child in n: - collectGenericParams(w, child) - else: - discard - proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addParLe sdefTag, trLineInfo(w, sym.infoImpl) dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo @@ -353,14 +339,12 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeLib(w, dest, sym.annexImpl) - # For routine symbols, pre-collect generic params into w.locals before writing - # the type. This ensures they get consistent short names, and their sdefs are - # written in the type where lazy loading can find them via extractLocalSymsFromTree. - if sym.kindImpl in routineKinds and sym.astImpl != nil and sym.astImpl.len > genericParamsPos: - inc w.inProc - collectGenericParams(w, sym.astImpl[genericParamsPos]) - dec w.inProc - + # Generic params are written as *global* symbols (with a module suffix) so that + # they get their own index entries and can be looked up lazily. This matters for + # generic routines that have a separate forward declaration and implementation: + # the two share the same generic param symbols, but each is serialized as its own + # index entry. If the params were local, a reference from the implementation's + # entry could not resolve the sdef emitted in the forward declaration's entry. writeType(w, dest, sym.typImpl) writeSym(w, dest, sym.ownerFieldImpl) # Store the AST for routine symbols and constants @@ -433,9 +417,10 @@ template withNode(w: var Writer; dest: var TokenBuf; n: PNode; body: untyped) = dest.addParRi proc addLocalSym(w: var Writer; n: PNode) = - ## Add symbol from a node to locals set if it's a symbol node - if n != nil and n.kind == nkSym and n.sym != nil and w.inProc > 0: - w.locals.incl(n.sym.itemId) + ## Previously forced proc-local symbols to be written without a module suffix. + ## All symbols are now emitted as global (see `isLocalSym`), so `w.locals` is + ## intentionally left empty. + discard proc addLocalSyms(w: var Writer; n: PNode) = case n.kind @@ -819,6 +804,8 @@ type symCounter: int32 index: Table[string, NifIndexEntry] # Simple embedded index for offsets suffix: string + contentStart: int # stream offset of the module body, so a full-AST load can + # rewind after lazy symbol loads moved the cursor DecodeContext* = object infos: LineInfoWriter @@ -827,11 +814,20 @@ type syms: Table[string, (PSym, NifIndexEntry)] mods: Table[FileIndex, NifModule] cache: IdentCache + mainModuleSuffix: string + ## Mangled module name of the module being compiled fresh (cmdM). Symbols + ## belonging to it that are re-exported by a dependency must NOT be loaded + ## as stubs, otherwise they collide with the freshly compiled originals. proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache) +proc setMainModule*(c: var DecodeContext; fileIdx: FileIndex) = + ## Records the module that is being compiled fresh so that re-exports of its + ## own symbols by dependencies are not turned into duplicate stubs. + c.mainModuleSuffix = modname(fileIdx.int, c.infos.config) + proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry; buf: var TokenBuf): Cursor = let s = addr c.mods[module].stream @@ -895,7 +891,10 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): "whose NIF file hasn't been written yet." var stream = nifstreams.open(modFile) let index = readEmbeddedIndex(stream) - c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix) + # `readEmbeddedIndex` leaves the cursor at the start of the module body. + let contentStart = offset(stream.r) + c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix, + contentStart: contentStart) proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry = let ii = addr c.mods[module].index @@ -927,8 +926,19 @@ proc createTypeStub(c: var DecodeContext; t: SymId): PType = if i < name.len and name[i] == '.': inc i let suffix = name.substr(i) let id = ItemId(module: moduleId(c, suffix).int32, item: itemId) - let offs = c.getOffset(id.module.FileIndex, name) - result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) + let ii = addr c.mods[id.module.FileIndex].index + let offs = ii[].getOrDefault(name) + if offs.offset == 0 and k == ord(tyNone): + # A `tyNone` placeholder (e.g. the type of a symbol-choice node) that is + # not present in its owning module's index. Such types are copied during + # template/generic instantiation in another module but keep their original + # owner, so the owner never serialised them. They carry no information, so + # synthesise a fresh, fully-loaded empty type instead of failing. + result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Complete) + else: + if offs.offset == 0: + raiseAssert "symbol has no offset: " & name + result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) c.types[name] = (result, offs) proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: string; @@ -1544,6 +1554,25 @@ proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix] else: raiseAssert "expected ParRi but got " & $tok.kind +proc addReexportedEnumFields(c: var DecodeContext; sym: PSym; interf: var TStrTable) = + ## When a non-pure enum type is (re-)exported, its fields must also become + ## visible (unqualified) to importers. In a from-source build this happens via + ## `rawImportSymbol`'s enum handling when the type is imported; the lazy IC + ## importer never runs that, so we materialise the fields into the interface + ## here, when the export list is processed. + loadSym(c, sym) + if sym.kindImpl != skType or sfPure in sym.flagsImpl: return + let et = sym.typImpl + if et == nil: return + loadType(c, et) + if et.kind notin {tyEnum, tyBool}: return + let fields = et.nImpl + if fields == nil: return + for i in 0 ..< fields.len: + let f = fields[i] + if f != nil and f.kind == nkSym and f.sym != nil: + strTableAdd(interf, f.sym) + proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; interf: var TStrTable; suffix: string; module: int): PrecompiledModule = result = PrecompiledModule(topLevel: newNode(nkStmtList)) @@ -1601,9 +1630,15 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; while true: if t.kind == Symbol: let symAsStr = pool.syms[t.symId] - let sym = resolveSym(c, symAsStr, false) - if sym != nil: - strTableAdd(interf, sym) + # Skip symbols that are re-exported by this dependency but actually + # belong to the module we are compiling fresh: loading them as stubs + # would shadow/collide with the freshly compiled originals. + if c.mainModuleSuffix.len == 0 or + parseSymName(symAsStr).module != c.mainModuleSuffix: + let sym = resolveSym(c, symAsStr, false) + if sym != nil: + strTableAdd(interf, sym) + addReexportedEnumFields(c, sym, interf) t = next(s) elif t.kind == ParRi: break @@ -1636,8 +1671,11 @@ proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHi let module = moduleId(c, string(suffix), flags) # Load the module AST (or just replay actions if loadFullAst is false) - # processTopLevel also collects export instructions + # processTopLevel also collects export instructions. + # Lazy symbol loading may have moved the stream cursor since the module was + # opened, so rewind to the start of the module body before reading it. let s = addr c.mods[module].stream + s[].r.jumpTo(c.mods[module].contentStart) var t = next(s[]) if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): t = next(s[]) # skip (stmts diff --git a/compiler/deps.nim b/compiler/deps.nim index cfe0f0975f..1d2378337a 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -302,6 +302,51 @@ proc whenMarkerHolds(c: DepContext; s: var Stream): bool = if v == "false": result = false # else (true / unknown ident): keep result +proc parseImportPath(s: var Stream; t: var PackedToken): seq[string] = + ## Parse an import path expression and return the list of module paths it + ## refers to. Handles plain idents (`foo`), string literals, `std/foo` + ## infixes (including nested ones like `std/private/since`) and bracketed + ## groups like `std/[bitops, fenv]` which expand to several imports. + ## On entry `t` is the first token of the expression; on exit `t` is the + ## token immediately following the whole expression. + result = @[] + case t.kind + of Ident: + result.add pool.strings[t.litId] + t = next(s) + of StringLit: + result.add pool.strings[t.litId] + t = next(s) + of ParLe: + let tag = pool.tags[t.tagId] + if tag == "infix": + t = next(s) # skip 'infix' tag + if t.kind == Ident: t = next(s) # skip the operator (`/`) + let left = parseImportPath(s, t) + let right = parseImportPath(s, t) + let prefix = if left.len == 1: left[0] else: "" + for r in right: + if prefix.len > 0: result.add prefix & "/" & r + else: result.add r + if t.kind == ParRi: t = next(s) # skip closing ')' + elif tag == "bracket": + t = next(s) # skip 'bracket' tag + while t.kind != ParRi and t.kind != EofToken: + result.add parseImportPath(s, t) + if t.kind == ParRi: t = next(s) # skip closing ')' + else: + # Unknown subtree: skip it entirely. + var depth = 1 + t = next(s) + while depth > 0 and t.kind != EofToken: + if t.kind == ParLe: inc depth + elif t.kind == ParRi: dec depth + if depth == 0: break + t = next(s) + if t.kind == ParRi: t = next(s) + else: + t = next(s) + proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) = ## Read a .deps.nif file and process imports/includes let depsPath = c.depsFile(pair) @@ -344,33 +389,32 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) = elif n.kind == EofToken: break t = next(s) continue - # Handle path expression (could be ident, string, or infix like std/foo) - var importPath = "" - if t.kind == Ident: - importPath = pool.strings[t.litId] - elif t.kind == StringLit: - importPath = pool.strings[t.litId] - elif t.kind == ParLe and pool.tags[t.tagId] == "infix": - # Handle std / foo style imports - t = next(s) # skip infix tag - if t.kind == Ident: # operator (/) - t = next(s) - if t.kind == Ident: # first part (std) - importPath = pool.strings[t.litId] - t = next(s) - if t.kind == Ident: # second part (foo) - importPath = importPath & "/" & pool.strings[t.litId] - if importPath.len > 0: - if tag == "include": - processInclude(c, importPath, current) - else: - processImport(c, importPath, current) - # Skip to end of node + # Process the path expression(s). Each path supports plain idents, + # string literals, `std/foo` infixes (possibly nested, e.g. + # `std/private/since`) and bracketed groups like `std/[bitops, fenv]` + # that expand to several imports. A plain `import a, b, c` lists several + # modules as siblings; a `fromimport` has a single path followed by the + # imported symbol list, which must not be treated as modules. + if tag == "fromimport": + for importPath in parseImportPath(s, t): + if importPath.len > 0: + processImport(c, importPath, current) + else: + while t.kind != ParRi and t.kind != EofToken: + for importPath in parseImportPath(s, t): + if importPath.len > 0: + if tag == "include": + processInclude(c, importPath, current) + else: + processImport(c, importPath, current) + # Drain any remaining tokens of this node (e.g. the symbol list of a + # `fromimport`), up to and including the node's closing ')'. var depth = 1 - while depth > 0: - t = next(s) + while depth > 0 and t.kind != EofToken: if t.kind == ParLe: inc depth elif t.kind == ParRi: dec depth + if depth == 0: break + t = next(s) else: # Skip unknown node var depth = 1 diff --git a/compiler/mangleutils.nim b/compiler/mangleutils.nim index 2ae9545184..edb11c1390 100644 --- a/compiler/mangleutils.nim +++ b/compiler/mangleutils.nim @@ -54,6 +54,19 @@ proc mangleParamExt*(s: PSym): string = proc mangleProcNameExt*(graph: ModuleGraph, s: PSym): string = result = "__" - result.add graph.ifaces[s.itemId.module].uniqueName + # Under incremental compilation the main module is registered at its source + # file index, but its symbols are keyed by the NIF-suffix file index, which has + # no `ifaces` slot. Only the main module has this gap (dependencies are + # registered at their suffix index), so omitting the unique name for it stays + # collision-free: the mangled base name plus `disamb` already disambiguate. + if s.itemId.module >= 0 and s.itemId.module < graph.ifaces.len: + result.add graph.ifaces[s.itemId.module].uniqueName result.add "_u" - result.addInt s.itemId.item # s.disamb # + # Use `disamb` rather than `itemId.item`: under incremental compilation a + # symbol loaded from a NIF file gets a fresh, load-order-dependent `itemId.item` + # (from the per-module symbol counter), which is neither stable across the + # processes that compile vs. use a module nor guaranteed distinct from another + # loaded symbol's. `disamb` is assigned deterministically per (module, name) + # and, together with the already-prepended mangled name, yields a unique and + # stable C identifier. + result.addInt s.disamb diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 3d094bd7b7..8fe918a0a1 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -375,6 +375,9 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx elif graph.config.cmd == cmdM: # For cmdM: load system.nim from NIF first, then compile the main module connectPipelineCallbacks(graph) + # Record the main module so the IC loader won't materialise duplicate stubs + # for its own symbols when a dependency (e.g. system) re-exports them. + setIcMainModule(projectFile) graph.config.m.systemFileIdx = fileInfoIdx(graph.config, graph.config.libpath / RelativeFile"system.nim") when not defined(nimKochBootstrap): diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 8009f7293c..2de2c0e163 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -512,6 +512,10 @@ proc semArrayIndex(c: PContext, n: PNode): PType = if c.inGenericContext > 0: result.incl tfUnresolved else: result = e.typ.skipTypes({tyTypeDesc}) + if result.state == Sealed: + # The index type was loaded from the IC cache and must not be mutated + # in place; work on a copy so we can mark it as an implicit static. + result = copyType(result, c.idgen, getCurrOwner(c)) result.incl tfImplicitStatic elif e.kind in (nkCallKinds + {nkBracketExpr}) and hasUnresolvedArgs(c, e): if not isOrdinalType(e.typ.skipTypes({tyStatic, tyAlias, tyGenericInst, tySink})): diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 5f846c80ac..ce026eac89 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -808,7 +808,11 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType, isInstValue = false): if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and t.elementType.n != nil: discard replaceObjBranches(cl, t.elementType.n) - elif result.n != nil and t.kind == tyObject: + elif result.n != nil and t.kind == tyObject and result.state != Sealed: + # A type loaded from the IC cache already had its object branches + # resolved when it was originally compiled, and must not be mutated in + # place (nor copied, which would break object-inheritance identity), so + # only non-Sealed types are processed here. # Invalidate the type size as we may alter its structure result.size = -1 result.n = replaceObjBranches(cl, result.n) @@ -860,7 +864,10 @@ proc recomputeFieldPositions*(t: PType; obj: PNode; currPosition: var int) = for i in 1..