diff --git a/compiler/ast.nim b/compiler/ast.nim index 48135c1984..068b3c4f86 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -77,6 +77,16 @@ proc backendEnsureMutable*(t: PType) {.inline.} = # ^ IC review this later if t.state == Partial: loadType(t) +proc unsealForTransform*(t: PType) {.inline.} = + ## The transformer/lambda lifting also run inside `nim m` when the VM + ## compiles a LOADED routine (macro evaluation, `getImpl`). Their mutations + ## are process-local — transformed bodies are never written back to a NIF — + ## so downgrade the loaded type to mutable, mirroring the `cmdNifC` loader + ## which loads everything `Complete` for exactly this reason (see + ## `ast2nif.loadedState`). + if t.state == Partial: loadType(t) + if t.state == Sealed: t.state = Complete + proc owner*(s: PSym): PSym {.inline.} = if s.state == Partial: loadSym(s) result = s.ownerFieldImpl @@ -1133,10 +1143,19 @@ proc copyType*(t: PType, idgen: IdGenerator, owner: PSym): PType = assignType(result, t) result.symImpl = t.sym # backend-info should not be copied -proc exactReplica*(t: PType): PType = +proc exactReplica*(t: PType; idgen: IdGenerator): PType = + ## Replica that KEEPS `itemId` — the generic-param binding tables + ## (`LayeredIdTable`) key on it, so the copy must keep matching its + ## original — but mints a FRESH `uniqueId`: uniqueId is the SERIALIZATION + ## identity (NIF type names key on it) and must be unique per instance. + ## Replicas sharing the original's uniqueId serialized as duplicate defs + ## under one NIF name; the loader collapsed them into a single type, + ## losing their flag differences (use-site `tfUnresolved` typedescs) or + ## their structure (meta instance bodies shadowing a generic's canonical + ## body). result = PType(kind: t.kind, ownerFieldImpl: t.owner, sizeImpl: defaultSize, alignImpl: defaultAlignment, itemId: t.itemId, - uniqueId: t.uniqueId) + uniqueId: nextTypeId(idgen)) assignType(result, t) result.symImpl = t.sym # backend-info should not be copied diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index be97c5d5b0..d054f3bab8 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -26,6 +26,8 @@ import typekeys import ic / [enum2nif] proc typeToNifSym(typ: PType; config: ConfigRef): string = + # NOTE: uniqueId is the serialization identity and is unique per instance — + # `exactReplica` keeps only itemId shared with its original (see ast.nim) assert not typ.uniqueId.isBackendMinted result = "`t" result.addInt ord(typ.kind) @@ -34,6 +36,16 @@ proc typeToNifSym(typ: PType; config: ConfigRef): string = result.add '.' result.add modname(typ.uniqueId.module, config) +proc icNifTypeName*(typ: PType; config: ConfigRef): string = + ## The serialized NIF name of a type, recorded next to RTTI data + ## definitions in the cnif artifact so a later run can re-demand the + ## typeinfo when a reused TU still references it (the def-retention + ## check). Backend-minted types have no NIF name. + if typ != nil and not typ.uniqueId.isBackendMinted: + result = typeToNifSym(typ, config) + else: + result = "" + proc toHookIndexEntry*(config: ConfigRef; typeId: ItemId; hookSym: PSym): HookIndexEntry = ## Converts a type ItemId and hook symbol to a HookIndexEntry for the NIF index. let typeSymName = "`t" & $typeId.item & "." & cachedModuleSuffix(config, typeId.module.FileIndex) @@ -164,6 +176,9 @@ type #writtenTypes: seq[PType] # types written in this module, to be unloaded later #writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later writtenPackages: HashSet[string] + writtenTypeNames: HashSet[string] # NIF names of type defs already emitted; + # `exactReplica` copies share the canonical + # type's uniqueId and thus its NIF name proc isLocalSym(sym: PSym): bool {.inline.} = ## Every symbol is emitted as a *global* (module-suffixed) name so that its @@ -299,6 +314,15 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = dest.addIntLit typ.alignImpl dest.addIntLit typ.paddingAtEndImpl dest.addIntLit typ.itemId.item # nonUniqueId + # `exactReplica` keeps the canonical type's itemId (binding-table key) + # while minting a fresh uniqueId (the NIF name): when the two halves + # name different modules, the loader cannot reconstruct itemId.module + # from the type's name — serialize it explicitly + if typ.itemId.module != typ.uniqueId.module and + not typ.itemId.isBackendMinted: + dest.addStrLit modname(typ.itemId.module, w.infos.config) + else: + dest.addDotToken writeType(w, dest, typ.typeInstImpl) #if typ.kind in {tyProc, tyIterator} and typ.nImpl != nil and typ.nImpl.kind != nkFormalParams: @@ -327,7 +351,17 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = # module (or nowhere), leaving dangling references (e.g. `symbol has no # offset` for a `pointer` type whose itemId.module drifted away). typ.state = Sealed - writeTypeDef(w, dest, typ) + let name = typeToNifSym(typ, w.infos.config) + if w.writtenTypeNames.containsOrIncl(name): + # BACKSTOP: uniqueId is unique per instance since `exactReplica` mints + # fresh ones, so two defs should never share a NIF name anymore. + # Should an id collision slip through regardless, duplicate defs are + # load-order POISON (the loader keys types by name; results.nim's + # Result body was once shadowed by a meta replica, silently nil-ing + # return types downstream) — degrade to a reference to the first def. + dest.addSymUse pool.syms.getOrIncl(name), NoLineInfo + else: + writeTypeDef(w, dest, typ) else: dest.addSymUse pool.syms.getOrIncl(typeToNifSym(typ, w.infos.config)), NoLineInfo @@ -536,12 +570,14 @@ proc moduleSuffix(conf: ConfigRef; f: FileIndex): string = proc trImport(w: var Writer; n: PNode) = for child in n: - if child.kind == nkSym: + if child.kind == nkSym and child.sym.kindImpl == skModule: + # a non-module sym appears for an `import v` inside an unexpanded + # template body (e.g. stew/importops' `when compiles((; import v))`): + # not a dependency edge, the import resolves at the expansion site w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) w.deps.addDotToken # flags w.deps.addDotToken # type let s = child.sym - assert s.kindImpl == skModule let fp = moduleSuffix(w.infos.config, s.positionImpl.FileIndex) w.deps.addStrLit fp # raw string literal, no wrapper needed w.deps.addParRi @@ -579,6 +615,7 @@ var repMethodTag = registerTag("repmethod") var includeTag = registerTag("include") var importTag = registerTag("import") var implTag = registerTag("implementation") +var reexpModTag = registerTag("reexpmod") proc registerNifAstTags*() = ## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag` @@ -607,6 +644,7 @@ proc registerNifAstTags*() = includeTag = registerTag("include") importTag = registerTag("import") implTag = registerTag("implementation") + reexpModTag = registerTag("reexpmod") proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n == nil: @@ -686,7 +724,15 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n[namePos].kind == nkSym: ast = n[namePos].sym.astImpl if ast == nil: ast = n - else: skipParams = true + else: + # params can only be recovered from `sym.typ.n` if the routine + # was actually semchecked. A routine nested in a TEMPLATE body + # (e.g. faststreams' `proc consumer(bytesVar: openArray[byte]) + # {.gensym.}` inside `consumeOutputs`) has a sym but a nil type — + # its params exist only in the AST; dropping them broke the + # template-param substitution at expansion ("undeclared + # identifier" for the injected name). + skipParams = n[namePos].sym.typImpl != nil w.withNode dest, ast: for i in 0 ..< ast.len: if i == paramsPos and skipParams: @@ -826,16 +872,20 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = # - import/include/export entries, `(replay ...)` macro-cache actions and the # rep* hook/converter/enumtostr registrations (all eagerly consumed by every # importer's sem via processTopLevel/loadTransitiveHooks). -# - every EXPORTED `(sd ...)`: full content for consts/types/vars/lets and for -# routines with inline semantics — templates, macros, iterators, generics -# (explicit `(genericparams)` in the routine ast or, for implicitly generic -# procs, `tyGenericParam`-kinded types in the signature) and `inline`-callconv -# procs. Plain procs/funcs/methods/converters hash the signature only, the -# body is skipped. (Nimony hashes generic bodies only via the `.inline` path; -# we close that gap here — generic bodies are instantiated by importers.) +# - every EXPORTED `(sd ...)`: full content for consts/types/vars/lets; for +# EVERY routine kind (plain procs, templates, macros, iterators, generics, +# `inline` procs alike) only the SIGNATURE — the body is skipped. A routine +# body is invisible to a dependent's SEM unless the dependent expands / +# instantiates / VM-runs it, and each of those records a NeedsImpl (strong) +# edge gating the dependent on this module's IMPL cookie instead (see +# `cookieSd`). This keeps the iface cookie body-insensitive, so a body edit +# re-sems only the modules that actually consumed that body — not every +# importer (the old model folded inline-semantics bodies into the iface +# cookie, re-semming all importers on any such body edit). # - nothing else: private defs and top-level init code are invisible to -# importers' sem (their effects on dependents' CODEGEN are covered by the -# nifc backend's transitive NIF-mtime invalidation, which is unchanged). +# importers' sem (their effects on dependents' CODEGEN — and the codegen +# effect of inline iterator/proc body edits — are covered by the nifc +# backend's transitive NIF-mtime invalidation, which is unchanged). # # Token-content hashing only — line infos never enter the hash. Names DEFINED # inside a hashed (sd) (params, locals, the embedded `(td `tK.item.mod)` defs) @@ -982,21 +1032,6 @@ proc hashRegion(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; updateAtom s, t inc i -proc scanSigTypeMarkers(buf: TokenBuf; start, theEnd: int): bool = - ## True if the routine's serialized signature marks it as inline-semantics: - ## an `inline` calling convention or a `tyGenericParam`-kinded type - ## (implicitly generic proc). - let gpPrefix = "`t" & $ord(tyGenericParam) & "." - var i = start - while i < theEnd: - let t = buf[i] - if t.kind == Ident and pool.strings[t.litId] == "inline": - return true - if t.kind in {Symbol, SymbolDef} and pool.syms[t.symId].startsWith(gpPrefix): - return true - inc i - result = false - proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): int = ## Contributes one `(sd ...)` subtree to the cookie; returns the index past it. result = nextTree(buf, start) @@ -1016,20 +1051,18 @@ proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): in kind = parse(TSymKind, pool.tags[buf[fields[0]].tagId]) var skipFrom = -1 var skipTo = -1 - if kind in {skProc, skFunc, skMethod, skConverter}: - var fullBody = scanSigTypeMarkers(buf, fields[7], fields[8]) + if kind in routineKinds: + # Routines contribute their SIGNATURE only to the iface cookie. A routine + # body is invisible to a dependent's SEM unless the dependent expands, + # instantiates, or VM-runs it — and each of those records a NeedsImpl + # (strong) edge that gates the dependent on this module's IMPL cookie + # instead (templates -> semTemplateExpr, generics -> generateInstance, + # macros/compile-time procs -> the VM's genProc, getImpl -> opcGetImpl). + # Inline iterators and `inline`-callconv procs are inlined at codegen; the + # nifc backend's transitive NIF-mtime invalidation re-codegens their users. + # So no routine body needs to live in the iface cookie. let ast = fields[9] - if not fullBody and buf[ast].kind == ParLe: - # routine ast tree: tag flags type name pattern genericParams params ... - var p = ast + 1 # flags atom - p = nextTree(buf, p) # -> type slot - p = nextTree(buf, p) # -> son 0 (name) - p = nextTree(buf, p) # -> son 1 (pattern) - p = nextTree(buf, p) # -> son 2 (genericParams) - if buf[p].kind == ParLe and - pool.tags[buf[p].tagId] == toNifTag(nkGenericParams): - fullBody = true - if not fullBody and buf[ast].kind == ParLe: + if buf[ast].kind == ParLe: # skip son `bodyPos` (6) of the routine ast tree; NOT the last element — # sem appends the result sym at `resultPos` (7) after the body. let astEnd = nextTree(buf, ast) @@ -1043,8 +1076,8 @@ proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): in if ok: skipFrom = p skipTo = nextTree(buf, p) - # templates/macros/iterators and all non-routine kinds (consts carry their - # value, types their structure incl. default field values): hash everything. + # non-routine kinds (consts carry their value, types their structure incl. + # default field values): hash everything. hashRegion(s, c, buf, start, result, skipFrom, skipTo, keepFirstDefLiteral = true) proc scanStmtsForCookie(s: var Sha1State; c: var CookieCtx; buf: TokenBuf) = @@ -1184,7 +1217,8 @@ proc writeEdgesFile(config: ConfigRef; thisModule: int32; implDeps: seq[int]) = proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; opsLog: seq[LogEntry]; replayActions: seq[PNode] = @[]; - implDeps: seq[int] = @[]) = + implDeps: seq[int] = @[]; + reexportedModules: seq[(string, string)] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var content = createTokenBuf(300) @@ -1205,6 +1239,17 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; var bottom = createTokenBuf(300) w.writeToplevelNode content, bottom, n + # Re-exported MODULES (`import x; export x`): semExport puts only x's + # member syms into the nkExportStmt; the module sym itself reaches the + # exporter's interface via `reexportSym` and acts as a QUALIFIER there + # (`asmm.x86.nd`). Serialize (name, suffix) pairs so the loader can + # rebuild that part of the interface. + for (mname, msuffix) in reexportedModules: + w.deps.addParLe reexpModTag, NoLineInfo + w.deps.addStrLit mname + w.deps.addStrLit msuffix + w.deps.addParRi + # the implTag is used to tell the loader that the # bottom of the file is the implementation of the module: content.addParLe implTag, NoLineInfo @@ -1556,7 +1601,8 @@ proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string; skip n result = loadSymStub(c, s, thisModule, localSyms) else: - raiseAssert "sym expected but got " & $n.kind + raiseAssert "sym expected but got " & $n.kind & ( + if n.kind == Ident: " '" & pool.strings[n.litId] & "'" else: "") proc isStub*(t: PType): bool {.inline.} = t.state == Partial proc isStub*(s: PSym): bool {.inline.} = s.state == Partial @@ -1620,6 +1666,13 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms loadField t.alignImpl loadField t.paddingAtEndImpl t.itemId = itemId(t.itemId.module, loadAtom(int32, n)) # nonUniqueId + if n.kind == StringLit: + # itemId.module differs from uniqueId.module (an `exactReplica` of a + # foreign type): restore the canonical module half + t.itemId = itemId(int32(moduleId(c, pool.strings[n.litId])), t.itemId.item) + inc n + elif n.kind == DotToken: + inc n t.typeInstImpl = loadTypeStub(c, n, localSyms) t.nImpl = loadNode(c, n, typesModule, localSyms) @@ -2007,12 +2060,53 @@ proc resolveGlobalSym*(c: var DecodeContext; symAsStr: string): PSym = if not fileExists(modFile): return nil result = resolveSym(c, symAsStr, true) +proc resolveGlobalType*(c: var DecodeContext; typeName: string): PType = + ## By-name resolution for the backend's def-retention check: a type NIF + ## name (see `typeToNifSym`) recorded next to an RTTI definition in a + ## `.c.nif` artifact is looked up in the current sem state. Returns nil + ## when the type no longer exists — including when its whole module + ## vanished from the program (`createTypeStub` asserts on both). + if not typeName.startsWith("`t"): return nil + # `t.. + var i = len("`t") + while i < typeName.len and typeName[i] in {'0'..'9'}: inc i + if i >= typeName.len or typeName[i] != '.': return nil + inc i + while i < typeName.len and typeName[i] in {'0'..'9'}: inc i + if i >= typeName.len or typeName[i] != '.': return nil + let suffix = typeName.substr(i+1) + if suffix.len == 0: return nil + let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string + if not fileExists(modFile): return nil + let module = moduleId(c, suffix) + if c.mods[module].index.getOrDefault(typeName).offset == 0: return nil + result = createTypeStub(c, pool.syms.getOrIncl(typeName)) + proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: FileIndex): PSym = ## Tries to resolve a compiler proc from a module by checking the NIF index. - ## Returns nil if the symbol doesn't exist. + ## Returns nil if the symbol doesn't exist. The NIF disamb is mint order, so + ## `name.0.` can be any of the overloads sharing the name — for `newSeq` it + ## is the generic magic, not the RTL proc (a refc build then demands codegen + ## of the generic and dies on `seq[T]`): enumerate the index entries with + ## this basename and pick the one that carries `sfCompilerProc`. + result = nil let suffix = moduleSuffix(c.infos.config, moduleFileIdx) - let symName = name & ".0." & suffix - result = resolveSym(c, symName, true) + let module = moduleId(c, suffix) + let prefix = name & "." + var candidates: seq[int] = @[] + for key in c.mods[module].index.keys: + if key.len > prefix.len and key.startsWith(prefix): + let sn = parseSymName(key) + if sn.name == name: + candidates.add sn.count + # the loads below can grow `c.mods` (symbols reference other modules), so + # resolve only after the index iteration is done + for count in candidates: + let sym = resolveSym(c, name & "." & $count & "." & suffix, true) + if sym != nil: + loadSym(c, sym) + if sfCompilerProc in sym.flagsImpl: + return sym proc loadLogOp(c: var DecodeContext; logOps: var seq[LogEntry]; s: var Stream; kind: LogEntryKind; op: TTypeAttachedOp; module: int): PackedToken = result = next(s) @@ -2067,6 +2161,8 @@ type deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of logOps*: seq[LogEntry] module*: PSym # set by modulegraphs.nim! + reexportedModules*: seq[(string, string)] # (name, suffix) of re-exported MODULE syms; + # materialized by modulegraphs.nim proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) = tok = next(s) # skip `(import` @@ -2190,6 +2286,24 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; t = skipTree(s) elif t.tagId == importTag: loadImport(c, s, result.deps, t) + elif t.tagId == reexpModTag: + # a re-exported MODULE: (reexpmod "name" "suffix"); the module sym + # is a qualifier in this module's interface — materialized by the + # caller (modulegraphs), which can register interface tables + t = next(s) + var mname = "" + var msuffix = "" + if t.kind == StringLit: + mname = pool.strings[t.litId] + t = next(s) + if t.kind == StringLit: + msuffix = pool.strings[t.litId] + t = next(s) + if t.kind != ParRi: + raiseAssert "expected ParRi in reexpmod entry of module " & suffix + t = next(s) + if mname.len > 0 and msuffix.len > 0: + result.reexportedModules.add (mname, msuffix) elif t.tagId == implTag: cont = false elif LoadFullAst in flags: diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 35d417fe94..feb5babeac 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -394,7 +394,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; n # variable. Thus, we create a temporary pointer variable instead. let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray if needsIndirect: - n.typ = n.typ.exactReplica + n.typ = n.typ.exactReplica(p.module.idgen) n.typ.incl tfVarIsPtr a = initLocExprSingleUse(p, n) a = withTmpIfNeeded(p, a, needsTmp) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 51dc364178..042941b745 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -416,6 +416,12 @@ proc getSimpleTypeDesc(m: BModule; typ: PType): Rope = m.typeCache[sig] = result proc pushType(m: BModule; typ: PType) = + when defined(icDbgRefc): + if typ.kind == tySequence and + typ.elementType.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyGenericParam: + echo "[icRefc] pushType seq-of-genericparam t=", typeToString(typ), + " itemId=", typ.itemId.module, ".", typ.itemId.item, " mod=", m.module.name.s + echo getStackTrace() for i in 0..high(m.typeStack): # pointer equality is good enough here: if m.typeStack[i] == typ: return @@ -1164,6 +1170,11 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes tyUserTypeClass, tyUserTypeClassInst, tyInferred: result = getTypeDescAux(m, skipModifier(t), check, kind) else: + when defined(icDbgRefc): + echo "[icRefc] getTypeDescAux ", t.kind, " t=", typeToString(t), + " origTyp=", typeToString(origTyp), " t.itemId=", t.itemId.module, ".", t.itemId.item, + " sym=", (if t.sym != nil: t.sym.name.s else: "nil"), + " owner=", (if t.owner != nil: t.owner.name.s else: "nil") internalError(m.config, "getTypeDescAux(" & $t.kind & ')') result = "" # fixes bug #145: @@ -1202,6 +1213,10 @@ proc finishTypeDescriptions(m: BModule) = var check = initIntSet() while i < m.typeStack.len: let t = m.typeStack[i] + when defined(icDbgRefc): + echo "[icRefc] finishTypeDescriptions[", i, "] mod=", m.module.name.s, + " t=", typeToString(t), " kind=", t.kind, + " itemId=", t.itemId.module, ".", t.itemId.item if optSeqDestructors in m.config.globalOptions and t.skipTypes(abstractInst).kind == tySequence: seqV2ContentType(m, t, check) else: @@ -1399,7 +1414,7 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; m.s[cfsStrData].addDeclWithVisibility(Private): m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType") if m.config.cmd == cmdNifC: - m.icDataDefs.add (name, "") + m.icDataDefs.add (name, icNifName(m, origType)) proc genTypeInfoAux(m: BModule; typ, origType: PType, name: Rope; info: TLineInfo) = @@ -1687,8 +1702,13 @@ proc declareNimType(m: BModule; name: string; str: Rope, module: int) = m.s[cfsTypeInit1].addArgument(hcrGlobal): m.s[cfsTypeInit1].add("\"" & str & "\"") else: + # cnif-mark the name: this extern declaration is the reference the + # def-retention check consults when the defining TU regenerates and + # the typeinfo cannot be re-demanded (type vanished) — the referencing + # TU must lose its reuse then instead of producing a link error + let declName = if m.config.cmd == cmdNifC: markCName(str) else: str m.s[cfsStrData].addDeclWithVisibility(Extern): - m.s[cfsStrData].addVar(kind = Local, name = str, typ = nr) + m.s[cfsStrData].addVar(kind = Local, name = declName, typ = nr) proc genTypeInfo2Name(m: BModule; t: PType): Rope = var it = t @@ -1828,7 +1848,7 @@ proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLin m.s[cfsStrData].addDeclWithVisibility(Private): m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2") if m.config.cmd == cmdNifC: - m.icDataDefs.add (name, "") + m.icDataDefs.add (name, icNifName(m, origType)) var flags = 0 if not canFormAcycle(m.g.graph, t): flags = flags or 1 @@ -1894,7 +1914,7 @@ proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineIn m.s[cfsStrData].addDeclWithVisibility(Private): m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2") if m.config.cmd == cmdNifC: - m.icDataDefs.add (name, "") + m.icDataDefs.add (name, icNifName(m, origType)) var flags = 0 if not canFormAcycle(m.g.graph, t): flags = flags or 1 @@ -2067,6 +2087,10 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = let marker = m.g.typeInfoMarker.getOrDefault(sig) if marker.str != "": + when defined(icDbgRefc): + if "catchableerror" in marker.str: + echo "[icNti] ", marker.str, " in mod=", m.module.name.s, + " -> extern:globalMarker owner=", marker.owner cgsym(m, "TNimType") cgsym(m, "TNimNode") declareNimType(m, "TNimType", marker.str, marker.owner) @@ -2077,7 +2101,15 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = result = "NTI$1$2_" % [rope(typeToC(t)), rope($sig)] m.typeInfoMarker[sig] = result + when defined(icDbgRefc): + template dbgNti(branch: string) = + if "catchableerror" in result: + echo "[icNti] ", result, " in mod=", m.module.name.s, " -> ", branch + else: + template dbgNti(branch: string) = discard + if m.config.cmd == cmdNifC and result in m.g.graph.icCachedDataDefs: + dbgNti "extern:cachedDataDefs" # already defined inside a reused TU from the previous run cgsym(m, "TNimType") cgsym(m, "TNimNode") @@ -2088,6 +2120,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = let old = m.g.graph.emittedTypeInfo.getOrDefault($result) if old != FileIndex(0): + dbgNti "extern:emittedTypeInfo" cgsym(m, "TNimType") cgsym(m, "TNimNode") declareNimType(m, "TNimType", result, old.int) @@ -2095,6 +2128,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = var owner = t.skipTypes(typedescPtrs).itemId.module if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner): + dbgNti "extern:ownerRouted" # make sure the type info is created in the owner module discard genTypeInfoV1(m.g.mods[owner], origType, info) # reference the type info as extern here @@ -2105,6 +2139,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = else: owner = m.module.position.int32 + dbgNti "DEFINED-HERE" m.g.typeInfoMarker[sig] = (str: result, owner: owner) #rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 1681e0a0cf..0549bca6b2 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -19,7 +19,7 @@ import mangleutils, cbuilderbase, modulegraphs from expanddefaults import caseObjDefaultBranch -from ast2nif import globalName, toNifFilename +from ast2nif import globalName, toNifFilename, icNifTypeName from typekeys import modname from std/algorithm import sort import cnif @@ -104,6 +104,15 @@ proc icNifName(m: BModule; s: PSym): string = else: result = "" +proc icNifName(m: BModule; t: PType): string = + ## The type flavor: recorded next to RTTI data definitions so the + ## def-retention check can re-demand the typeinfo of a regenerating TU's + ## previous artifact (`genTypeInfo` is type-driven, not symbol-driven). + if m.config.cmd == cmdNifC: + result = icNifTypeName(t, m.config) + else: + result = "" + proc redirectToLiveModule(m: BModule, q: BModule): BModule = ## A module whose cached translation unit is reused never generates code, ## so a definition that `findPendingModule` routes into it must be emitted diff --git a/compiler/cnif.nim b/compiler/cnif.nim index 0e402dd947..7868cd86c4 100644 --- a/compiler/cnif.nim +++ b/compiler/cnif.nim @@ -73,12 +73,14 @@ proc stripCnifMarks*(s: string): string = inc i const - CnifVersion* = "3" + CnifVersion* = "4" ## Artifact format version, stored in the meta head. Artifacts written ## by an older compiler lack the NIF names and the cref group the - ## def-retention check needs (v2) or the cdeps group the fine-grained - ## reuse gate needs (v3); `readCnifHeads` reports them as invalid so - ## their TUs simply regenerate once. + ## def-retention check needs (v2), the cdeps group the fine-grained + ## reuse gate needs (v3), or the type NIF names and cnif-marked extern + ## RTTI references the typeinfo flavor of the def-retention check + ## needs (v4); `readCnifHeads` reports them as invalid so their TUs + ## simply regenerate once. proc cnifDefDirective*(name, flags, nifName: string): string = CnifDefStart & name & CnifDefSep & flags & CnifDefSep & nifName & CnifDefEnd diff --git a/compiler/commands.nim b/compiler/commands.nim index 9e9349f1d1..3a8ae18559 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -653,6 +653,18 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = var key = "" var val = "" + # Record config-file switches so the `nim ic` driver can serialise them into a + # precompiled-config artifact and have its per-module child processes replay + # them instead of re-parsing the `nim.cfg` chain (and re-running `config.nims` + # in the VM) on every invocation. Only `passPP` (config-file) switches are + # captured; command-line switches are forwarded by the build graph as usual. + # Path-search switches are skipped: their net effect already lives in the + # resolved `searchPaths` the driver forwards as `--path`, and replaying their + # raw (often relative-to-config-dir) arguments here would misresolve. + if pass == passPP and switch.normalize notin + ["path", "p", "nimblepath", "lazypath", "excludepath", + "nonimblepath", "clearnimblepath", "nimcache"]: + conf.icConfigSwitches.add (switch, arg) case switch.normalize of "eval": expectArg(conf, switch, arg, pass, info) @@ -937,6 +949,17 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectArg(conf, switch, arg, pass, info) if pass in {passCmd2, passPP}: conf.icGroup.incl(canonicalizePath(conf, AbsoluteFile arg).string) + of "icproject": + # `nim m`/`nim nifc` only: the ORIGINAL project file (see options.icProject) + expectArg(conf, switch, arg, pass, info) + if pass in {passCmd2, passPP}: + conf.icProject = canonicalizePath(conf, AbsoluteFile arg).string + of "icpreparsedconfig": + # `nim m`/`nim nifc` only: path of the precompiled-config artifact (see + # options.icPreparsedConfig). Read in `passCmd1`, before `loadConfigs`, so + # config loading can replay it instead of re-parsing the `nim.cfg` chain. + expectArg(conf, switch, arg, pass, info) + conf.icPreparsedConfig = arg of "import": expectArg(conf, switch, arg, pass, info) if pass in {passCmd2, passPP}: diff --git a/compiler/deps.nim b/compiler/deps.nim index 005e683c35..ca3019a806 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -10,8 +10,8 @@ ## Generate a .build.nif file for nifmake from a Nim project. ## This enables incremental and parallel compilation using the `m` switch. -import std / [os, tables, sets, times, osproc, algorithm, strtabs] -import options, msgs, lineinfos, pathutils, condsyms +import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio] +import options, msgs, lineinfos, pathutils, condsyms, icconfig import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder] import "../dist/nimony/src/gear2" / modnames @@ -415,13 +415,22 @@ proc parseImportPath(s: var Stream; t: var PackedToken): seq[string] = let tag = pool.tags[t.tagId] if tag == "infix": t = next(s) # skip 'infix' tag - if t.kind == Ident: t = next(s) # skip the operator (`/`) + var op = "" + if t.kind == Ident: + op = pool.strings[t.litId] + t = next(s) let left = parseImportPath(s, t) let right = parseImportPath(s, t) - let prefix = if left.len == 1: left[0] else: "" - for r in right: - if prefix.len > 0: result.add prefix & "/" & r - else: result.add r + if op == "as": + # `import ../rlp/results as rlp_results`: the alias is not a path + # component — treating `as` like `/` produced the garbage path + # `../rlp/results/rlp_results`, silently dropping the dependency + result = left + else: + let prefix = if left.len == 1: left[0] else: "" + for r in right: + if prefix.len > 0: result.add prefix & "/" & r + else: result.add r if t.kind == ParRi: t = next(s) # skip closing ')' elif tag == "prefix": # Relative import paths: `import ../dist/checksums/...` parses as @@ -634,6 +643,21 @@ proc generateBuildFile(c: DepContext): string = # buckets (and rejects calls as ambiguous that multi-dispatch accepts) if optMultiMethods in c.config.globalOptions: forwardedArgs.add "--multimethods:on" + # the children compile each MODULE as their own project file, which makes + # that module's package the "main package" and unfilters foreign-package + # diagnostics — a vendored package's hintAsError/warningAsError promotions + # then abort builds the whole-program compilation accepts. Forward the + # real project so children filter diagnostics identically. + forwardedArgs.add "--icproject:" & c.config.projectFull.string + # Precompiled config: serialise the driver's config once and have every + # child replay it instead of re-parsing the `nim.cfg` chain and re-running + # `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig` + # restores the old per-child config parsing (for bisecting a suspected + # config-replay divergence without clearing caches). + if not isDefined(c.config, "icNoPreparsedConfig"): + let cfgArtifact = nimcache / "ic_config.cfg.nif" + writeIcConfig(c.config, cfgArtifact) + forwardedArgs.add "--icPreparsedConfig:" & cfgArtifact # Define nifler command b.addTree "cmd" @@ -857,8 +881,16 @@ proc commandIc*(conf: ConfigRef) = rawMessage(conf, errGenerated, "project file not found: " & projectFile) return - # Create nimcache directory - createDir(getNimcacheDir(conf).string) + # Create nimcache directory; start from a clean one when its format + # stamp is absent or outdated (see `icFormatVersion`) + let cacheDir = getNimcacheDir(conf).string + createDir(cacheDir) + let versionFile = cacheDir & "/ic.version" + let stamp = if fileExists(versionFile): readFile(versionFile) else: "" + if stamp != icFormatVersion: + removeDir(cacheDir) + createDir(cacheDir) + writeFile(versionFile, icFormatVersion) var c = DepContext( config: conf, @@ -901,19 +933,56 @@ proc commandIc*(conf: ConfigRef) = # Process dependencies traverseDeps(c, rootPair, rootNode) - # Generate build file - let buildFile = generateBuildFile(c) - rawMessage(conf, hintSuccess, "generated: " & buildFile) + # Discovery loop: imports GENERATED by macros (chronicles builds + # `import chronicles/textlines` via parseStmt from the chronicles_sinks + # define) are invisible to the static scanner. A failing `nim m` child + # records "missing-path \t importer-path" in icmissing.txt; we add the + # module (and an edge from its importer) to the graph and rerun — + # nifmake's mtime pruning keeps completed work. + let missingFile = getNimcacheDir(conf).string & "/icmissing.txt" + removeFile missingFile + var rounds = 0 + while true: + # Generate build file + let buildFile = generateBuildFile(c) + rawMessage(conf, hintSuccess, "generated: " & buildFile) - # Automatically run nifmake - let nifmake = findNifmake() - if nifmake.len == 0: - rawMessage(conf, hintSuccess, "run: nifmake run " & buildFile) - else: + # Automatically run nifmake + let nifmake = findNifmake() + if nifmake.len == 0: + rawMessage(conf, hintSuccess, "run: nifmake run " & buildFile) + break let cmd = quoteShell(nifmake) & " run " & quoteShell(buildFile) rawMessage(conf, hintExecuting, cmd) let exitCode = execShellCmd(cmd) - if exitCode != 0: + if exitCode == 0: break + var discovered = false + inc rounds + if rounds <= 20 and fileExists(missingFile): + for line in lines(missingFile): + let parts = line.split('\t') + if parts.len != 2 or parts[0].len == 0: continue + let pair = c.toPair(parts[0]) + let importerIdx = c.processedModules.getOrDefault( + c.toPair(parts[1]).modname, -1) + var idx = c.processedModules.getOrDefault(pair.modname, -1) + if idx == -1: + let newNode = Node(files: @[pair], id: c.nodes.len) + if c.systemNodeId >= 0: + newNode.deps.add c.systemNodeId + c.processedModules[pair.modname] = newNode.id + c.nodes.add newNode + idx = newNode.id + traverseDeps(c, pair, newNode) + discovered = true + if importerIdx >= 0 and idx >= 0 and idx notin c.nodes[importerIdx].deps: + # the build-graph edge the scanner could not see: forces the + # discovered module to be built before its importer re-sems + c.nodes[importerIdx].deps.add idx + discovered = true + removeFile missingFile + if not discovered: rawMessage(conf, errGenerated, "nifmake failed with exit code: " & $exitCode) + break else: rawMessage(conf, errGenerated, "nim ic not available in bootstrap build") diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 4610f89a8d..9e00179fbb 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -307,6 +307,7 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) = elif not (owner.typ.isClosure or owner.isNimcall and not owner.isExplicitCallConv or isEnv): localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" % [s.name.s, owner.name.s, $owner.typ.callConv]) + unsealForTransform(owner.typ) incl(owner.typ, tfCapturesEnv) if not isEnv: owner.typ.callConv = ccClosure diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 645956de57..a588a8ff06 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -378,6 +378,9 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; conflictsWith: TLineInfo, note = errGenerated) = ## Emit a redefinition error if in non-interactive mode if c.config.cmd != cmdInteractive: + when defined(icDbgRefc): + echo "[icRedef] ", s + echo getStackTrace() localError(c.config, info, note, "redefinition of '$1'; previous declaration here: $2" % [s, c.config $ conflictsWith]) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index c908ce41d0..2a68ca74e7 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -105,18 +105,39 @@ type # they were in its previous artifact and a # reused TU still references them (the # backend def-migration check) + icPreserveTypeInfos*: Table[int, seq[PType]] # the same for RTTI data + # definitions, which are type-driven: the + # type whose `genTypeInfo` the TU must + # re-demand icImplDeps*: IntSet # NeedsImpl edge tracking under `nim m`: # module ids (FileIndex) whose routine BODIES - # this compilation consumed at compile time - # (VM-compiled or getImpl'ed). Written to the - # `.edges` sidecar; deps.nim then gates the - # dependent on those modules' IMPL cookie - # instead of the iface cookie, so e.g. - # `const x = dep.foo()` re-sems when foo's - # body changes. Bodies with inline semantics - # (templates/macros/generics/iterators) need - # no tracking: they are part of the iface - # cookie itself. + # this compilation consumed at compile time. + # Written to the `.edges` sidecar; deps.nim + # then gates the dependent on those modules' + # IMPL cookie instead of the iface cookie, so + # e.g. `const x = dep.foo()` re-sems when foo's + # body changes. Uniform across body-access + # kinds — the iface cookie hashes signatures + # ONLY (see ast2nif.cookieSd), so every body + # consumer records an edge here: VM-compiled / + # getImpl'ed bodies (recordIcImplDep from vm/ + # vmgen), expanded templates (semTemplateExpr) + # and instantiated generics (generateInstance). + # Inline iterators / `inline` procs are NOT + # tracked: they are inlined at codegen, where + # the nifc backend's NIF-mtime invalidation + # already re-codegens their users. + icQualIfaces*: IntSet # module positions whose interface tables were + # populated ONLY for qualified access through a + # module re-export (`import x; export x`); the + # Iface.module stays nil so a later direct + # import still takes the full load path + inVMTransform*: int # >0 while the VM compiles a routine body + # (vmgen.genProc's transformBody): hooks lifted + # there (e.g. for closure-env types of LOADED + # routines) are process-local VM artifacts — + # serializing them would embed references to + # derived env-field syms that no module defines packageSyms*: TStrTable deps*: IntSet # the dependency graph or potentially its transitive closure. @@ -285,6 +306,18 @@ iterator allSyms*(g: ModuleGraph; m: PSym): PSym = if s != nil: yield s +proc reexportedModuleSyms*(g: ModuleGraph; m: PSym): seq[(string, string)] = + ## (name, NIF module suffix) of MODULE syms in `m`'s interface — these are + ## re-exports (`import x; export x`, added by `reexportSym`) acting as + ## qualifiers (`m.x.sym`). Consumed by the NIF writer; semExport does not + ## put them into the nkExportStmt children, so the AST walk cannot see them. + result = @[] + var seen = initIntSet() + for s in g.ifaces[m.position].interf.data: + if s != nil and s.kind == skModule and s.position != m.position and + not seen.containsOrIncl(s.position): + result.add (s.name.s, cachedModuleSuffix(g.config, FileIndex s.position)) + proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym = let importHidden = optImportHidden in m.options result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name) @@ -346,6 +379,15 @@ proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; # Key-based deduplication for opsLog: different type objects (e.g. canon vs # orig) can have different itemIds but the same structural key. let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback) + if g.inVMTransform > 0 and g.config.cmd == cmdM: + # hook lifted while the VM compiles a routine body (closure-env types of + # loaded routines): register it for in-process lookup but keep it out of + # the serialized log — it is a process-local artifact whose type graph + # references derived env-field syms that no module's NIF defines + if g.loadedOps[op].getOrDefault(key) == nil: + g.loadedOps[op][key] = value + g.attachedOps[op][t.itemId] = value + return let existing = g.loadedOps[op].getOrDefault(key) if existing == nil: # Stamp the entry with the module whose compilation produced the hook @@ -941,6 +983,39 @@ when not defined(nimKochBootstrap): registerLoadedHooks(g, precomp.logOps) for d in precomp.deps: stack.add d + proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym = + ## A re-exported MODULE (`import x; export x`) acts as a qualifier in the + ## re-exporting module's interface (`asmm.x86.nd`). Reconstruct a module + ## symbol for it and make its interface tables available for qualified + ## lookup (`someSym` reads `g.ifaces[position]`) — WITHOUT registering + ## the module: `Iface.module` stays nil so a later direct import still + ## takes the full load path (replayStateChanges etc.). + var isKnown = false + let fIdx = g.config.registerNifSuffix(msuffix, isKnown) + if fIdx.int >= g.ifaces.len: setLen(g.ifaces, fIdx.int + 1) + if g.ifaces[fIdx.int].module != nil and + g.ifaces[fIdx.int].module.name.s == mname: + # properly registered already (directly imported earlier): reuse it + return g.ifaces[fIdx.int].module + result = PSym(kindImpl: skModule, itemId: itemId(int32(fIdx), 0'i32), + name: getIdent(g.cache, mname), + infoImpl: newLineInfo(fIdx, 1, 1), + positionImpl: int(fIdx)) + setOwner(result, getPackage(g.config, g.cache, fIdx)) + if g.ifaces[fIdx.int].module == nil and + not g.icQualIfaces.containsOrIncl(fIdx.int): + var interf = initStrTable() + var interfHidden = initStrTable() + let precomp = loadNifModule(ast.program, ModuleSuffix(msuffix), + interf, interfHidden, {}) + # chains: the re-exported module may itself re-export modules + for (n2, s2) in precomp.reexportedModules: + let inner = materializeReexportedModule(g, n2, s2) + if inner != nil: + strTableAdd(interf, inner) + g.ifaces[fIdx.int].interf = interf + g.ifaces[fIdx.int].interfHidden = interfHidden + proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex; flags: set[LoadFlag] = {}): PrecompiledModule = ## Returns 'nil' if the module needs to be recompiled. @@ -966,6 +1041,10 @@ when not defined(nimKochBootstrap): g.ifaces[fileIdx.int].interf, g.ifaces[fileIdx.int].interfHidden, flags) result.module = m + for (mname, msuffix) in result.reexportedModules: + let ms = materializeReexportedModule(g, mname, msuffix) + if ms != nil: + strTableAdd(g.ifaces[fileIdx.int].interf, ms) # Mark module as cached g.cachedMods.incl fileIdx.int diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index bfe2fcf9d1..752b3be13d 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -149,11 +149,30 @@ proc enforceDefRetention(g: ModuleGraph; mainPos: int; # targets the un-reuse fallback. Demand-side dedup (`declaredThings`, # the cached/claim shortcuts) makes redundant re-demands cheap. clear g.icPreserveDefs + clear g.icPreserveTypeInfos var unreuse = initHashSet[int]() for src in sources.items: template check(defseq) = for d in defseq: if d.cname notin cachedDefs: + if d.nifname.startsWith("`t"): + # an RTTI definition: re-demand is type-driven (`genTypeInfo`), + # there is no symbol to resolve + let typ = resolveGlobalType(ast.program, d.nifname) + if typ != nil: + g.icPreserveTypeInfos.mgetOrPut(src.target, @[]).add typ + if icDebug: + stderr.writeLine "[icRetain] preserve typeinfo " & d.cname + continue + elif d.cname in refdBy: + # type vanished: un-reuse the TUs that still reference it + for tu in refdBy[d.cname]: unreuse.incl tu + if icDebug: + stderr.writeLine "[icRetain] cannot re-demand typeinfo " & + d.cname & "; un-reusing referencing TUs" + continue + else: + continue var sym: PSym = nil if d.nifname.len > 0: sym = resolveGlobalSym(ast.program, d.nifname) @@ -406,6 +425,16 @@ proc eagerHookCandidate(sym: PSym): bool = let pt = typ.n[i].typ if pt == nil: return false if iterOverType(pt, isMetaIter, nil): return false + # a `=dup` of an imported type returns it by value; for "lying" importc + # typedefs like `jmp_buf` (declared as `object`, really a C array) that + # signature does not compile. Demand-driven codegen never demands such + # sem-bookkeeping hooks (under refc nothing dups a `C_JmpBuf`), and no + # working artifact can call one — its prototype would be the same + # invalid C — so they are safe to skip. + let ret = typ.returnType + if ret != nil: + let r = ret.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct}) + if r.sym != nil and sfImportc in r.sym.flags: return false true proc finishModule(g: ModuleGraph; bmod: BModule) = @@ -472,6 +501,9 @@ proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) = if g.icPreserveDefs.hasKey(moduleId): for sym in g.icPreserveDefs[moduleId]: requestAnyDef(bmod, sym) + if g.icPreserveTypeInfos.hasKey(moduleId): + for t in g.icPreserveTypeInfos[moduleId]: + discard genTypeInfo(g.config, bmod, t, unknownLineInfo) proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = ## Main entry point for NIF-based C code generation. diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 5d31dea57f..9c5e4c6d07 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -11,7 +11,7 @@ import llstream, commands, msgs, lexer, ast, - options, idents, wordrecg, lineinfos, pathutils, scriptconfig + options, idents, wordrecg, lineinfos, pathutils, scriptconfig, icconfig import std/[os, strutils, strtabs] @@ -246,6 +246,12 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = setDefaultLibpath(conf) + # `nim ic` children replay the precompiled config the driver recorded once, + # instead of re-reading the `nim.cfg` chain and re-running `config.nims` in the + # VM. A missing/format-incompatible artifact returns false: fall through to + # normal config loading so an older child or a deleted cache still works. + if conf.icPreparsedConfig.len > 0 and applyIcConfig(conf, conf.icPreparsedConfig): + return template readConfigFile(path) = let configPath = path conf.currentConfigDir = configPath.splitFile.dir.string diff --git a/compiler/options.nim b/compiler/options.nim index 434f30f47f..70bbb415e8 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -29,6 +29,18 @@ const nimEnableCovariance* = defined(nimEnableCovariance) + icFormatVersion* = "2" + ## Version of the IC cache format (the sem-NIF module layout written by + ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever + ## that layout changes: `commandIc` wipes a nimcache whose `ic.version` + ## stamp differs, instead of letting a newer reader mis-parse records + ## written by an older compiler (nifmake's rebuild check is mtime-only + ## and knows nothing about format changes). + ## v2: iface cookie hashes routine SIGNATURES only (no inline-semantics + ## body folding); body access now records a NeedsImpl edge instead. A v1 + ## cache mixes body-sensitive and body-insensitive cookies, so it must be + ## wiped rather than warm-rebuilt. + type # please make sure we have under 32 options # (improves code efficiency a lot!) TOption* = enum # **keep binary compatible** @@ -386,6 +398,24 @@ type # recursion resolves in-memory) and each gets its NIF # written, instead of being loaded from a precompiled # NIF. See `compiler/deps.nim` (SCC grouping). + icProject*: string # under `nim m`/`nim nifc`: absolute path of the + # ORIGINAL project file. The child's own project file + # is the module being compiled, which would make that + # module's package the "main package" and unfilter + # foreign-package diagnostics; the real project + # restores whole-program filtering semantics. + icPreparsedConfig*: string # under `nim m`/`nim nifc`: path of the precompiled + # config artifact written once by the `nim ic` driver. + # When set, `loadConfigs` replays the recorded + # config-file switches from it instead of re-reading + # the `nim.cfg` chain and re-running `config.nims` + # (which the VM makes expensive) per subprocess. + icConfigSwitches*: seq[tuple[switch, arg: string]] + # the config-file (`passPP`) switches applied while + # loading config, in order. Recorded by every nim + # process; only the `ic` driver serialises them. + # Path-search switches are excluded — the driver + # forwards the resolved `searchPaths` as `--path`. spellSuggestMax*: int # max number of spelling suggestions for typos cppDefines*: HashSet[string] # (*) diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index c4db260a22..69fc3fd722 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -270,7 +270,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator var implDeps: seq[int] = @[] for id in graph.icImplDeps: implDeps.add id writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, - replayActions, implDeps) + replayActions, implDeps, reexportedModuleSyms(graph, module)) result = true @@ -304,6 +304,18 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF let precomp = moduleFromNifFile(graph, fileIdx) if precomp.module == nil: let nifPath = toNifFilename(graph.config, fileIdx) + # Record the miss for `nim ic`'s discovery loop: imports GENERATED + # by macros (e.g. chronicles' parseStmt("import chronicles/textlines"), + # driven by the chronicles_sinks define) are invisible to the static + # dependency scanner. The parent reads this file, adds the module — + # plus an edge from this importer — to the build graph and reruns. + try: + let f = open(getNimcacheDir(graph.config).string & "/icmissing.txt", fmAppend) + f.writeLine(toFullPath(graph.config, fileIdx) & "\t" & + graph.config.projectFull.string) + f.close() + except IOError, OSError: + discard globalError(graph.config, unknownLineInfo, "nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) & " (expected: " & nifPath & ")") @@ -382,7 +394,14 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx conf.projectMainIdx2 = projectFile - let packSym = getPackage(graph, projectFile) + var packSym = getPackage(graph, projectFile) + if graph.config.cmd in {cmdM, cmdNifC} and graph.config.icProject.len > 0: + # per-module IC children: the process' project file is the MODULE being + # compiled, which would make its package the "main package" and unfilter + # foreign-package diagnostics (a vendored package's hintAsError promotion + # then aborts builds the whole-program compilation accepts). Use the + # original project, forwarded by deps.nim via --icproject. + packSym = getPackage(graph, fileInfoIdx(graph.config, AbsoluteFile graph.config.icProject)) graph.config.mainPackageId = packSym.getPackageId graph.importStack.add projectFile diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 32c98cdb31..c837cad54e 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -373,6 +373,13 @@ proc addConverter*(c: PContext, conv: PSym) = proc addConverterDef*(c: PContext, conv: PSym) = addConverter(c, conv) + # record the definition for IC: the loader rebuilds Iface.converters from + # the NIF's (repconverter ...) entries (moduleFromNifFile); without the log + # entry a loaded module's converters were invisible to importers and + # implicit conversions silently stopped matching (e.g. faststreams' + # InputStreamHandle -> InputStream at toml_serialization call sites) + c.graph.opsLog.add LogEntry(kind: ConverterEntry, module: c.module.position, + key: "", sym: conv) proc addPureEnum*(c: PContext, e: PSym) = assert e != nil diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 74d3751900..3bff50186e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -27,6 +27,11 @@ const proc semTemplateExpr(c: PContext, n: PNode, s: PSym, flags: TExprFlags = {}; expectedType: PType = nil): PNode = rememberExpansion(c, n.info, s) + # IC: this expands `s`'s body into the current module's sem, so the module + # depends on that body — record a NeedsImpl (strong) edge to `s`'s module. + # The iface cookie hashes only signatures now, so a template body edit moves + # only the impl cookie, and just the modules that expanded it re-sem. + recordIcImplDep(c.graph, s) let info = getCallLineInfo(n) markUsed(c, info, s) onUse(info, s) @@ -57,6 +62,16 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = elif {efWantStmt, efAllowStmt} * flags != {}: result.typ = newTypeS(tyVoid, c) else: + when defined(icDbgRefc): + echo "[icNoType] semOperand: ", renderTree(result, {renderNoComments}), + " kind=", result.kind, + (if result.kind in {nkCall, nkCommand} and result[0].kind == nkSym: + " calleeTyp=" & (if result[0].sym.typ == nil: "NIL" else: + $result[0].sym.typ.kind & " ret=" & + (if result[0].sym.typ.returnType == nil: "NIL" + else: $result[0].sym.typ.returnType.kind)) + else: "") + echo getStackTrace() localError(c.config, n.info, errExprXHasNoType % renderTree(result, {renderNoComments})) result.typ = errorType(c) @@ -83,6 +98,17 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType if result.typ == nil and efInTypeof in flags: result.typ = c.voidType elif result.typ == nil or result.typ == c.enforceVoidContext: + when defined(icDbgRefc): + echo "[icNoType] semExprWithType: ", renderTree(result, {renderNoComments}), + " kind=", result.kind, + (if result.kind in {nkCall, nkCommand} and result[0].kind == nkSym: + " callee=" & result[0].sym.name.s & + " calleeTyp=" & (if result[0].sym.typ == nil: "NIL" else: + $result[0].sym.typ.kind & " ret=" & + (if result[0].sym.typ.returnType == nil: "NIL" + else: $result[0].sym.typ.returnType.kind)) + else: "") + echo getStackTrace() localError(c.config, n.info, errExprXHasNoType % renderTree(result, {renderNoComments})) result.typ = errorType(c) @@ -2144,6 +2170,12 @@ proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode = if c.p.owner.kind notin {skMacro, skTemplate} and c.p.resultSym != nil and c.p.resultSym.typ.isMetaType: + when defined(icDbgRefc): + echo "[icMetaRet] meta result type for ", c.p.owner.name.s, ": ", + typeToString(c.p.resultSym.typ), " kind=", c.p.resultSym.typ.kind, + " flags=", c.p.resultSym.typ.flags, + " uid=", c.p.resultSym.typ.uniqueId.module, ".", c.p.resultSym.typ.uniqueId.item, + " state=", c.p.resultSym.typ.state if isEmptyType(result.typ): # we inferred a 'void' return type: c.p.resultSym.typ = errorType(c) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 02abe3510c..7b758bdbbc 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -119,11 +119,44 @@ proc freshGenSyms(c: PContext; n: PNode, owner, orig: PSym, symMap: var SymMappi proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind) +proc aliasLoadedTypedescParams(c: PContext, instantiated, orig: PSym): bool = + ## When the generic being instantiated had its body LOADED from a NIF (only + ## `nim m`/`nim nifc`, only for a generic owned by another module), that body + ## re-sems from plain identifiers — ast2nif serialises locals/params as idents, + ## not `nkSym`. A `T: typedesc[...]` param referenced as a type must then + ## resolve `T` to the bound type, but the instantiated skParam carries the + ## concrete type `instantiateProcType` typedesc-skipped it to, which an ident + ## lookup cannot use as a type name. Shadow each such param with an `skType` + ## alias of the same name in a fresh scope layer (the alias is exactly how Nim + ## models "this name denotes a type"). In-process bodies reach the param as + ## `nkSym` and never take this path, hence the command gate. + ## + ## Returns true iff a scope layer was opened; the caller must `closeScope`. + if c.config.cmd notin {cmdM, cmdNifC} or orig == nil or + orig.itemId.module == c.module.position or + orig.typ == nil or orig.typ.n == nil: + return false + result = false + let procParams = instantiated.typ.n + for i in 1.. ", typeToString(paramType), + " (kind=", paramType.kind, ")" # The default value is instantiated and fitted against the final # concrete param type. We avoid calling `replaceTypeVarsN` on the @@ -382,6 +422,11 @@ proc generateInstance(c: PContext, fn: PSym, pt: LayeredIdTable, ## parameters to their concrete types within the generic instance. # no need to instantiate generic templates/macros: internalAssert c.config, fn.kind notin {skMacro, skTemplate} + # IC: instantiating `fn` consumes its generic body in the current module's + # sem — record a NeedsImpl (strong) edge to `fn`'s module. The iface cookie + # hashes only signatures now, so a generic body edit moves only the impl + # cookie, and just the modules that instantiated it re-sem. + recordIcImplDep(c.graph, fn) # generates an instantiated proc if c.instCounter > 50: globalError(c.config, info, "generic instantiation too nested") diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 6dcbeabc5c..ecd2cb7260 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1361,7 +1361,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, for i in 0.. ", + (if result != nil: typeToString(result) else: "MISS"), + " allowMeta=", cl.allowMetaTypes if result == nil: if cl.allowMetaTypes or tfRetType in t.flags: return localError(cl.c.config, t.sym.info, "cannot instantiate: '" & typeToString(t) & "'") @@ -401,7 +415,7 @@ proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType = proc instCopyType*(cl: var TReplTypeVars, t: PType): PType = # XXX: relying on allowMetaTypes is a kludge if cl.allowMetaTypes: - result = t.exactReplica + result = t.exactReplica(cl.c.idgen) else: result = copyType(t, cl.c.idgen, t.owner) copyTypeProps(cl.c.graph, cl.c.idgen.module, result, t) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 5f53d7ef44..d33f4001b6 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -135,6 +135,11 @@ proc put(c: var TCandidate, key, val: PType) {.inline.} = writeStackTrace() if c.c.module.name.s == "temp3": echo "binding ", key, " -> ", val + when defined(icDbgRefc): + if key.kind in {tyGenericParam, tyTypeDesc}: + echo "[icBind] put ", key.kind, " ", typeToString(key), " uid=", key.uniqueId.module, ".", + key.uniqueId.item, " itemId=", key.itemId.module, ".", key.itemId.item, + " state=", key.state, " -> ", typeToString(val) put(c.bindings, key, val.skipIntLit(c.c.idgen)) proc typeRel*(c: var TCandidate, f, aOrig: PType, @@ -911,7 +916,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType = case typ.kind of tyStatic: param = paramSym skConst - param.typ = typ.exactReplica + param.typ = typ.exactReplica(m.c.idgen) #copyType(typ, c.idgen, typ.owner) if typ.n == nil: param.typ.incl tfInferrableStatic @@ -919,7 +924,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType = param.ast = typ.n of tyFromExpr: param = paramSym skVar - param.typ = typ.exactReplica + param.typ = typ.exactReplica(m.c.idgen) #copyType(typ, c.idgen, typ.owner) else: param = paramSym skType @@ -972,7 +977,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType = if ff.kind == tyUserTypeClassInst: result = generateTypeInstance(c, m.bindings, typeClass.sym.info, ff) else: - result = ff.exactReplica + result = ff.exactReplica(m.c.idgen) #copyType(ff, c.idgen, ff.owner) result.n = checkedBody @@ -2666,7 +2671,7 @@ proc staticAwareTypeRel(m: var TCandidate, f: PType, arg: var PNode): TTypeRelat # The ast of the type does not point to the symbol. # Without this we will never resolve a `static proc` with overloads let copiedNode = copyNode(arg) - copiedNode.typ = exactReplica(copiedNode.typ) + copiedNode.typ = exactReplica(copiedNode.typ, m.c.idgen) copiedNode.typ.n = arg arg = copiedNode typeRel(m, f, arg.typ) diff --git a/compiler/typekeys.nim b/compiler/typekeys.nim index ca2f1467e5..435b927e3a 100644 --- a/compiler/typekeys.nim +++ b/compiler/typekeys.nim @@ -153,7 +153,12 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef for a in t.sonsImpl: c.typeKey a, flags, conf of tyDistinct: - if CoDistinct in flags: + if t.sonsImpl.len == 0: + # a bare `distinct` typeclass (e.g. `foo(distinct, ...)` matched + # against a `T: type` param) has no base type to key — it IS its kind + withTree c.m, toNifTag(t.kind): + c.m.addEmpty() + elif CoDistinct in flags: if t.symImpl != nil: symKey(c, t.symImpl, conf) if t.symImpl == nil or tfFromGeneric in t.flagsImpl: c.typeKey t.sonsImpl[^1], flags, conf @@ -238,9 +243,14 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef if t.typeInstImpl != nil: # prevent against infinite recursions here, see bug #8883: let inst = t.typeInstImpl + if inst.state == Partial: + # a lazily-loaded typeInst stub has no sons until forced in + assert c.tl != nil + c.tl(inst) t.typeInstImpl = nil # IC: spurious writes are ok since we set it back immediately assert inst.kind == tyGenericInst - c.typeKey inst.sonsImpl[0], flags, conf + if inst.sonsImpl.len > 0: + c.typeKey inst.sonsImpl[0], flags, conf for i in 1..