diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index c4bab071ee..5a8ed60ab0 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -176,6 +176,7 @@ type #writtenTypes: seq[PType] # types written in this module, to be unloaded later #writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later writtenPackages: HashSet[string] + depSuffixes: HashSet[string] # module suffixes already emitted as `(import ...)` deps proc isLocalSym(sym: PSym): bool {.inline.} = ## Every symbol is emitted as a *global* (module-suffixed) name so that its @@ -376,9 +377,21 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = # module scope and a template's open/mixin symbol of the same name resolves to # the field instead of a local, producing "type mismatch: got 'T'". Fields are # still indexed (for `obj.field` resolution via the loaded object type); they - # are merely not advertised as importable. `skEnumField` stays importable — - # enum values are legitimately usable as bare identifiers. - if sym.kindImpl != skField and {sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}: + # are merely not advertised as importable. Plain `skEnumField` stays importable + # — enum values are legitimately usable as bare identifiers — but a field of a + # `{.pure.}` enum is NOT: the source path keeps pure fields out of the importer + # scope (`declarePureEnumField`), reachable only qualified or via the restricted + # pure-enum mechanism (`importPureEnumFields`, fed by `ifaces[].pureEnums` which + # a loaded module rebuilds from its `PureEnumEntry` log ops). Marking them + # bare-importable made a loaded pure enum's fields leak into module scope + # (`populateInterfaceTablesFromIndex` adds every `x`/Exported sym to `interf`), + # e.g. nim-json-serialization's pure `JsonValueKind.Number` shadowing web3's + # `Number = distinct uint64` so `uint64(x).Number` failed under `nim ic` + # ("undeclared field 'Number'"). + let isPureEnumField = sym.kindImpl == skEnumField and sym.typImpl != nil and + sym.typImpl.symImpl != nil and sfPure in sym.typImpl.symImpl.flagsImpl + if sym.kindImpl != skField and not isPureEnumField and + {sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}: dest.addIdent "x" else: dest.addDotToken @@ -546,6 +559,7 @@ proc trImport(w: var Writer; n: PNode) = let fp = moduleSuffix(w.infos.config, s.positionImpl.FileIndex) w.deps.addStrLit fp # raw string literal, no wrapper needed w.deps.addParRi + w.depSuffixes.incl fp proc trExport(w: var Writer; n: PNode) = # Collect export information for the index @@ -576,11 +590,13 @@ var repTraceTag = registerTag("reptrace") var repDeepCopyTag = registerTag("repdeepcopy") var repEnumToStrTag = registerTag("repenumtostr") var repMethodTag = registerTag("repmethod") +var repPureEnumTag = registerTag("reppureenum") #var repClassTag = registerTag("repclass") var includeTag = registerTag("include") var importTag = registerTag("import") var implTag = registerTag("implementation") var reexpModTag = registerTag("reexpmod") +var offerTag = registerTag("offer") proc registerNifAstTags*() = ## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag` @@ -606,10 +622,12 @@ proc registerNifAstTags*() = repDeepCopyTag = registerTag("repdeepcopy") repEnumToStrTag = registerTag("repenumtostr") repMethodTag = registerTag("repmethod") + repPureEnumTag = registerTag("reppureenum") includeTag = registerTag("include") importTag = registerTag("import") implTag = registerTag("implementation") reexpModTag = registerTag("reexpmod") + offerTag = registerTag("offer") proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n == nil: @@ -726,8 +744,23 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = writeNode(w, dest, ast[i], forAst) dec w.inProc of nkImportStmt: - # this has been transformed for us, see `importer.nim` to contain a list of module syms: - trImport w, n + if w.inProc > 0: + # An `import` inside a template/macro/proc body — e.g. stew/importops' + # `tryImport`: `when compiles((; import v)): import v`. It is part of the + # body AST and must be serialized as a real node so the template + # re-expands it at each use site; it is NOT a module-level dependency + # edge (the import resolves where the template expands, against that + # module's deps). Diverting it to `w.deps` (the top-level path below) + # dropped it entirely: its child is the unexpanded template parameter + # `v`, not a module sym, so `trImport` wrote nothing and the body + # round-tripped EMPTY — a NIF-loaded `tryImport` then imported nothing. + w.withNode dest, n: + for i in 0 ..< n.len: + writeNode(w, dest, n[i], forAst) + else: + # top-level import: recorded as a dependency edge — `importer.nim` has + # already transformed `n` to contain a list of module syms. + trImport w, n of nkIncludeStmt: trInclude w, n of nkExportStmt, nkExportExceptStmt: @@ -821,6 +854,11 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() + of PureEnumEntry: + content.addParLe repPureEnumTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) + content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) + content.addParRi() of GenericInstEntry: discard "will only be written later to ensure it is materialized" @@ -1202,7 +1240,11 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; opsLog: seq[LogEntry]; replayActions: seq[PNode] = @[]; implDeps: seq[int] = @[]; - reexportedModules: seq[(string, string)] = @[]) = + reexportedModules: seq[(string, string)] = @[]; + genericOffers: seq[tuple[generic, inst: PSym; + concreteTypes: seq[PType]; + genericParamsCount: int]] = @[]; + resolvedImportDeps: seq[FileIndex] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var content = createTokenBuf(300) @@ -1223,6 +1265,25 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; var bottom = createTokenBuf(300) w.writeToplevelNode content, bottom, n + # Resolved import edges that left no syntactic `import` node in the top-level + # AST: an import generated INSIDE a `when` condition (e.g. stew/importops' + # `when tryImport x:` -> `when compiles((; import x)): import x`) really + # imports `x` — `addImportFileDep` recorded the edge in `graph.importDeps` — + # but the import node is folded away with the condition, so `trImport` never + # saw it and the NIF `deps` section omitted it. The backend closure walk + # (nifbackend.loadBackendModules) follows NIF `deps`, so without this edge a + # template-imported module's `{.compile.}`/`{.passL.}` directives never replay + # and its C/asm objects go unlinked (undefined `hashtree_hash`/`my_c_add` at + # link). Emit any resolved edge not already written as a syntactic import. + for f in resolvedImportDeps: + let fp = moduleSuffix(config, f) + if not w.depSuffixes.containsOrIncl(fp): + w.deps.addParLe importTag, NoLineInfo + w.deps.addDotToken # flags + w.deps.addDotToken # type + w.deps.addStrLit fp + w.deps.addParRi + # Re-exported MODULES (`import x; export x`): semExport puts only x's # member syms into the nkExportStmt; the module sym itself reaches the # exporter's interface via `reexportSym` and acts as a QUALIFIER there @@ -1234,6 +1295,24 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; w.deps.addStrLit msuffix w.deps.addParRi + # Generic-instance OFFERS: every generic instance this module created + # (`getOrDefault[MultiCodec]`, …). A consumer that re-instantiates the same + # generic must REUSE this instance instead of re-running `instantiateBody` in + # its own module scope — which lacks symbols visible only at the generic's + # definition site (e.g. a distinct type's `==` from the type's module), so + # operator/mixin resolution would fail ("type mismatch" at `hashcommon.rawGet`). + # The loader (modulegraphs.moduleFromNifFile) rebuilds `procInstCache` from + # these so `genericCacheGet` hits and the wrong-scope re-instantiation is + # skipped. Layout: (offer ...). + for off in genericOffers: + w.deps.addParLe offerTag, NoLineInfo + w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.generic)), NoLineInfo + w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.inst)), NoLineInfo + w.deps.addIntLit off.genericParamsCount + for ct in off.concreteTypes: + w.deps.addSymUse pool.syms.getOrIncl(typeToNifSym(ct, w.infos.config)), NoLineInfo + w.deps.addParRi + # the implTag is used to tell the loader that the # bottom of the file is the implementation of the module: content.addParLe implTag, NoLineInfo @@ -1448,6 +1527,33 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]) +proc tryCreateTypeStub(c: var DecodeContext; t: SymId): PType = + ## Like `createTypeStub` but returns nil instead of raising when the type has + ## no offset in its module index (used by the best-effort `(offer …)` loader). + let name = pool.syms[t] + if not name.startsWith("`t"): return nil + result = c.types.getOrDefault(name)[0] + if result == nil: + var i = len("`t") + var k = 0 + while i < name.len and name[i] in {'0'..'9'}: + k = k * 10 + name[i].ord - ord('0') + inc i + if i < name.len and name[i] == '.': inc i + var itemVal = 0'i32 + while i < name.len and name[i] in {'0'..'9'}: + itemVal = itemVal * 10'i32 + int32(name[i].ord - ord('0')) + inc i + if i < name.len and name[i] == '.': inc i + let suffix = name.substr(i) + let id = itemId(moduleId(c, suffix).int32, itemVal) + let ii = addr c.mods[id.module.FileIndex].index + let offs = ii[].getOrDefault(name) + if offs.offset == 0: + return nil + result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) + c.types[name] = (result, offs) + proc createTypeStub(c: var DecodeContext; t: SymId): PType = let name = pool.syms[t] assert name.startsWith("`t") @@ -2128,6 +2234,11 @@ type module*: PSym # set by modulegraphs.nim! reexportedModules*: seq[(string, string)] # (name, suffix) of re-exported MODULE syms; # materialized by modulegraphs.nim + genericOffers*: seq[tuple[generic, inst: PSym; concreteTypes: seq[PType]; + genericParamsCount: int]] + ## generic instances this module created; modulegraphs.nim rebuilds + ## `procInstCache` from them so a consumer reuses the instance instead of + ## re-instantiating it in its own (operator-blind) module scope. proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) = tok = next(s) # skip `(import` @@ -2172,6 +2283,12 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; var t = next(s) # skip dot var cont = true let exportTag = pool.tags.getOrIncl"export" + # Top-level `let`/`var` sections are loaded even without LoadFullAst: they may + # declare `{.compileTime.}` globals whose VM slots the importer initializes + # eagerly (pipelines.initLoadedCompileTimeGlobals), which needs them visible in + # `topLevel`. They sit in the module header before `(implementation)`. + let letTag = pool.tags.getOrIncl(toNifTag(nkLetSection)) + let varTag = pool.tags.getOrIncl(toNifTag(nkVarSection)) while cont and t.kind != EofToken: if t.kind == ParLe: if t.tagId == replayTag: @@ -2210,6 +2327,8 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; t = loadLogOp(c, result.logOps, s, EnumToStrEntry, attachedTrace, module) elif t.tagId == repMethodTag: t = loadLogOp(c, result.logOps, s, MethodEntry, attachedTrace, module) + elif t.tagId == repPureEnumTag: + t = loadLogOp(c, result.logOps, s, PureEnumEntry, attachedTrace, module) #elif t.tagId == repClassTag: # t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module) elif t.tagId == exportTag: @@ -2269,10 +2388,39 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; t = next(s) if mname.len > 0 and msuffix.len > 0: result.reexportedModules.add (mname, msuffix) + elif t.tagId == offerTag: + # (offer ...) — see the + # writer. Resolve to PSyms/PTypes here; modulegraphs registers them into + # `procInstCache`. Best-effort: a type that fails to resolve drops the + # whole offer (the consumer then re-instantiates, the prior behaviour). + t = next(s) # skip (offer + var genSym, instSym: PSym = nil + var paramsCount = 0 + var cts: seq[PType] = @[] + var idx = 0 + var ok = true + while t.kind != ParRi and t.kind != EofToken: + if t.kind == Symbol: + if idx == 0: genSym = resolveHookSym(c, t.symId) + elif idx == 1: instSym = resolveHookSym(c, t.symId) + else: + let ct = tryCreateTypeStub(c, t.symId) + if ct == nil: ok = false + else: cts.add ct + inc idx + elif t.kind == IntLit: + paramsCount = int(pool.integers[t.intId]) + t = next(s) + if t.kind != ParRi: + raiseAssert "expected ParRi in offer entry of module " & suffix + t = next(s) + if ok and genSym != nil and instSym != nil: + result.genericOffers.add (genSym, instSym, cts, paramsCount) elif t.tagId == implTag: cont = false - elif LoadFullAst in flags: - # Parse the full statement + elif LoadFullAst in flags or t.tagId == letTag or t.tagId == varTag: + # Parse the full statement. let/var sections are loaded unconditionally + # (see above) so `{.compileTime.}` globals reach the eager initializer. var buf = createTokenBuf(50) nextSubtree(s, buf, t) t = next(s) # skip ParRi diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 5857531c8d..67f7b02fa5 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -986,7 +986,8 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode = type LogEntryKind* = enum - HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry + HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry, + PureEnumEntry LogEntry* = object kind*: LogEntryKind op*: TTypeAttachedOp diff --git a/compiler/commands.nim b/compiler/commands.nim index f5434fb621..31a185d838 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -508,6 +508,7 @@ proc parseCommand*(command: string): Command = of "jsonscript": cmdJsonscript of "nifc": cmdNifC # generate C from NIF files of "ic": cmdIc # generate .build.nif for nifmake + of "icconfig": cmdIcConfig # produce the precompiled config artifact else: cmdUnknown proc setCmd*(conf: ConfigRef, cmd: Command) = @@ -960,6 +961,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; # config loading can replay it instead of re-parsing the `nim.cfg` chain. expectArg(conf, switch, arg, pass, info) conf.icPreparsedConfig = arg + of "icconfigout": + # `nim icconfig` only: where to write the precompiled config artifact (see + # options.icConfigOut). The `nim ic` driver spawns the producer with this. + expectArg(conf, switch, arg, pass, info) + conf.icConfigOut = arg of "icbackendstage": # `nim nifc` only: per-module backend stage, one of cg|merge|emit (see # options.icBackendStage). Empty (switch unused) keeps the whole-program diff --git a/compiler/deps.nim b/compiler/deps.nim index 34d06ac0e1..08a272aa6b 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -11,8 +11,8 @@ ## This enables incremental and parallel compilation using the `m` switch. import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio] -import options, msgs, lineinfos, pathutils, condsyms, icconfig, - modulepaths, extccomp, cnif +import options, msgs, lineinfos, pathutils, condsyms, + modulepaths, extccomp, cnif, platform import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder] import "../dist/nimony/src/gear2" / modnames @@ -196,8 +196,11 @@ proc resolveInclude(c: DepContext; origin, toResolve: string): string = proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) -proc processInclude(c: var DepContext; includePath: string; current: Node) = - let resolved = resolveInclude(c, current.files[current.files.len - 1].nimFile, includePath) +proc processInclude(c: var DepContext; includePath: string; current: Node; origin: string) = + # `origin` = the file the `include` literally appears in (an included file's + # own nested includes/imports must resolve relative to IT, not the importing + # module's main file). + let resolved = resolveInclude(c, origin, includePath) if resolved.len == 0 or not fileExists(resolved): return @@ -211,8 +214,25 @@ proc processInclude(c: var DepContext; includePath: string; current: Node) = traverseDeps(c, c.toPair(resolved), current) discard c.includeStack.pop() -proc processImport(c: var DepContext; importPath: string; current: Node) = - let resolved = resolveImport(c, current.files[0].nimFile, importPath) +proc getsImplicitImports(c: DepContext; nimFile: string): bool = + ## Mirror the compiler's `belongsToStdlib` guard (pipelines.nim): `--import:X` + ## (conf.implicitImports) is applied only to NON-stdlib modules. The scanner + ## must agree, otherwise it edges a stdlib module → X that the compiler never + ## actually creates, fabricating a cycle that folds X — and the modules X + ## claims to produce — into the system SCC (whose `nim m` is driven from + ## system.nim and never reaches them). Stdlib == under conf.libpath. + not isRelativeTo(nimFile, c.config.libpath.string) + +proc processImport(c: var DepContext; importPath: string; current: Node; origin: string) = + # `origin` = the file the `import` literally appears in. Crucial for imports + # inside `include`d files: e.g. `system.nim` includes `system/excpt.nim`, which + # does `import stacktraces` — that must resolve relative to `excpt.nim` + # (lib/system/) → `lib/system/stacktraces.nim`, NOT relative to `system.nim` + # (lib/) which has no `stacktraces.nim`. Resolving against the main file silently + # dropped the `system → stacktraces` edge, so stacktraces was a separate SCC in + # the static round and got re-grouped (and recompiled with divergent type ids) + # only after the post-sem `.s.deps` revealed the edge. + let resolved = resolveImport(c, origin, importPath) if resolved.len == 0 or not fileExists(resolved): return @@ -226,12 +246,15 @@ proc processImport(c: var DepContext; importPath: string; current: Node) = # Every module depends on system.nim if c.systemNodeId >= 0: newNode.deps.add c.systemNodeId - # ... and on every `--import`ed module (conf.implicitImports). A `--import`ed - # module is itself imported by its own closure (which also gets these edges), - # so the cycle folds into one strongly-connected component (see computeSCCs), - # just like system.nim's closure. - for impId in c.implicitNodeIds: - if impId != newNode.id: newNode.deps.add impId + # ... and on every `--import`ed module (conf.implicitImports), but only for + # the non-stdlib modules the compiler actually applies implicit imports to + # (see getsImplicitImports). A `--import`ed module is imported by its own + # non-stdlib closure (which also gets these edges), so that cycle folds into + # one small strongly-connected component (see computeSCCs) instead of being + # smeared across system + stdlib. + if getsImplicitImports(c, pair.nimFile): + for impId in c.implicitNodeIds: + if impId != newNode.id: newNode.deps.add impId c.processedModules[pair.modname] = newNode.id c.nodes.add newNode traverseDeps(c, pair, newNode) @@ -251,10 +274,44 @@ proc skipSubtree(s: var Stream; first: PackedToken) = elif t.kind == ParRi: dec depth elif t.kind == EofToken: return -proc evalCondIdent(c: DepContext; v: string): bool = - ## Truth value of a bare identifier appearing in a `when` condition. +type + CondVal = enum + ## Tri-state truth of a `when` condition as the static scanner sees it. + ## `cvUnknown` is the crucial state: the scanner can't determine the value + ## (an arbitrary call like `compiles`/`tryImport`, an unknown const ident, + ## an unresolvable comparison). A dependency scanner must NEVER drop a real + ## import, so callers treat `cvUnknown` as "keep the dependency". The bug + ## this replaces: everything-unknown collapsed to `true`, and `not true` + ## is `false`, so an `else:` branch (emitted as `when (not COND)`) silently + ## dropped its imports (e.g. `when tryImport x: ... else: import x`, or + ## system's `else: include excpt` hiding `import stacktraces`). + cvFalse, cvTrue, cvUnknown + +proc toCondVal(b: bool): CondVal = (if b: cvTrue else: cvFalse) + +proc condNot(a: CondVal): CondVal = + case a + of cvFalse: cvTrue + of cvTrue: cvFalse + of cvUnknown: cvUnknown + +proc condAnd(a, b: CondVal): CondVal = + if a == cvFalse or b == cvFalse: cvFalse + elif a == cvTrue and b == cvTrue: cvTrue + else: cvUnknown + +proc condOr(a, b: CondVal): CondVal = + if a == cvTrue or b == cvTrue: cvTrue + elif a == cvFalse and b == cvFalse: cvFalse + else: cvUnknown + +proc evalCondIdent(c: DepContext; v: string): CondVal = + ## Truth value of a bare identifier appearing in a `when` condition. Unknown + ## idents are `cvUnknown` (kept), not `true` — so `when not SOMEIDENT:` no + ## longer drops its import. case v - of "false": false + of "true": cvTrue + of "false": cvFalse of "hasThreadSupport": # system.nim's `hasThreadSupport` is `compileOption("threads") and # not defined(nimscript)`; the conservative `true` would schedule the @@ -262,12 +319,12 @@ proc evalCondIdent(c: DepContext; v: string): bool = # whose NIFs a --threads:off compile never produces — nifmake then # sees missing outputs and re-runs the system rule (and everything # downstream) on every rerun. - optThreads in c.config.globalOptions + toCondVal(optThreads in c.config.globalOptions) of "usesDestructors": # system.nim's `usesDestructors = defined(gcDestructors) or # defined(gcHooks)`; guards mmdisp.nim's `include "system/gc"` whose # transitive imports (sharedlist, locks) an orc compile never produces. - isDefined(c.config, "gcDestructors") or isDefined(c.config, "gcHooks") + toCondVal(isDefined(c.config, "gcDestructors") or isDefined(c.config, "gcHooks")) of "isMainModule": # Only the project main module is compiled with `isMainModule` true; an # imported module's `when isMainModule` blocks are dead. The conservative @@ -275,171 +332,123 @@ proc evalCondIdent(c: DepContext; v: string): bool = # `tools/grammar_nanny`, a node that gets a cg rule but is never linked, # so the merge stage can pick it as a shared def's owner -> undefined # symbols at link). - c.scanningMain - else: true + toCondVal(c.scanningMain) + else: cvUnknown -proc evalCondExpr(c: DepContext; s: var Stream): bool = - ## Read exactly one condition expression from `s` and return its truth - ## value. Consumes tokens whether the expression is recognised or not so - ## the caller stays in sync. Recognises `defined(IDENT)`, the boolean - ## operators `not`/`and`/`or`, and the literals `true`/`false`. Anything - ## else (e.g. a call to an arbitrary proc) is treated as `true` — the - ## conservative direction, since a false negative here drops a real - ## dependency from the build graph. +proc constIdentValue(c: DepContext; ident: string): string = + ## String value of a compile-time platform constant that appears in `when` + ## guards, or "" when unknown. Mirrors the compiler's magics so the scanner + ## evaluates e.g. `when hostOS == "standalone"` the SAME way the real compile + ## does. Without this the comparison is "unknown" → the conservative `true`, + ## which is WRONG once negated (`else:` branches emit `not (==)`), so a real + ## conditional `include`/`import` is dropped (e.g. system's `else: include + ## excpt`, hiding `import stacktraces`). + # Must match the compiler's magics EXACTLY, incl. case: `hostOS`/`hostCPU` etc. + # fold to the lower-cased platform name (see semfold.nim mHostOS/mHostCPU), and + # user code compares against lower-case literals (`when hostOS == "linux"`). + case ident + of "hostOS": result = toLowerAscii(platform.OS[c.config.target.targetOS].name) + of "hostCPU": result = toLowerAscii(platform.CPU[c.config.target.targetCPU].name) + of "buildOS": result = toLowerAscii(platform.OS[c.config.target.hostOS].name) + of "buildCPU": result = toLowerAscii(platform.CPU[c.config.target.hostCPU].name) + else: result = "" + +proc readOperandValue(c: DepContext; s: var Stream): string = + ## Read one operand of an `==`/`!=` infix and return its string value (a string + ## literal verbatim, a platform-constant ident resolved, anything else ""), fully + ## consuming the operand (subtrees are skipped) so the caller stays in sync. let t = next(s) case t.kind + of StringLit: result = pool.strings[t.litId] + of Ident: result = constIdentValue(c, pool.strings[t.litId]) + of ParLe: + result = "" + skipSubtree(s, t) + else: result = "" + +proc evalCondCmp(c: DepContext; s: var Stream; isEq: bool): CondVal = + ## Evaluate `a == b` / `a != b`. Both operands known → real result; otherwise + ## `cvUnknown` (so a negated comparison keeps, not drops, the dependency). + let v1 = readOperandValue(c, s) + let v2 = readOperandValue(c, s) + if v1.len > 0 and v2.len > 0: + result = toCondVal((v1 == v2) == isEq) + else: + result = cvUnknown + +proc evalCondExpr(c: DepContext; s: var Stream; t: PackedToken): CondVal + +proc readCond(c: DepContext; s: var Stream): CondVal = + ## Read one full condition subtree (its own opener included) and evaluate it. + let t = next(s) + evalCondExpr(c, s, t) + +proc evalCondExpr(c: DepContext; s: var Stream; t: PackedToken): CondVal = + ## Evaluate the condition whose opening token `t` has ALREADY been read, + ## consuming the rest of the expression so the caller stays in sync. + ## Recognises `defined(IDENT)`, `not`/`and`/`or`, `==`/`!=` and the literals + ## `true`/`false`; everything else (an arbitrary call such as `compiles` / + ## `tryImport`, an unknown const) is `cvUnknown`. Both negation-sensitive + ## (`not cvUnknown == cvUnknown`) and short-circuit-free: `and`/`or` always + ## read both operands so the stream stays in sync regardless of the result. + case t.kind of Ident: result = evalCondIdent(c, pool.strings[t.litId]) of ParLe: let tag = pool.tags[t.tagId] + # For prefix/infix/call nodes the operator name is the first child; for a + # bare `(not ...)`/`(and ...)`/`(or ...)`/`(par ...)` node the tag itself is + # the operator and the operands follow directly. + var name = tag case tag of "call", "cmd", "callstrlit", "infix", "prefix": - # First child is the head (function/operator name). let head = next(s) - var name = "" if head.kind == Ident: name = pool.strings[head.litId] - case name - of "defined": - let arg = next(s) - var sym = "" - if arg.kind == Ident: sym = pool.strings[arg.litId] - result = sym.len > 0 and isDefined(c.config, sym) - of "not": - result = not evalCondExpr(c, s) - of "and": - result = evalCondExpr(c, s) - if result: result = evalCondExpr(c, s) - else: skipSubtree(s, next(s)) - of "or": - result = evalCondExpr(c, s) - if not result: result = evalCondExpr(c, s) - else: skipSubtree(s, next(s)) - else: - result = true - # Drain whatever remains until the matching ParRi. - var depth = 1 - while depth > 0: - let n = next(s) - if n.kind == ParLe: inc depth - elif n.kind == ParRi: dec depth - elif n.kind == EofToken: return + else: name = "" + else: discard + case name + of "defined": + let arg = next(s) + var sym = "" + if arg.kind == Ident: sym = pool.strings[arg.litId] + result = toCondVal(sym.len > 0 and isDefined(c.config, sym)) of "not": - result = not evalCondExpr(c, s) - var depth = 1 - while depth > 0: - let n = next(s) - if n.kind == ParLe: inc depth - elif n.kind == ParRi: dec depth - elif n.kind == EofToken: return + result = condNot(readCond(c, s)) of "and": - result = evalCondExpr(c, s) - if result: result = evalCondExpr(c, s) - else: skipSubtree(s, next(s)) - # consume closing ParRi - var depth = 1 - while depth > 0: - let n = next(s) - if n.kind == ParLe: inc depth - elif n.kind == ParRi: dec depth - elif n.kind == EofToken: return + let a = readCond(c, s) + let b = readCond(c, s) + result = condAnd(a, b) of "or": - result = evalCondExpr(c, s) - if not result: result = evalCondExpr(c, s) - else: skipSubtree(s, next(s)) - var depth = 1 - while depth > 0: - let n = next(s) - if n.kind == ParLe: inc depth - elif n.kind == ParRi: dec depth - elif n.kind == EofToken: return + let a = readCond(c, s) + let b = readCond(c, s) + result = condOr(a, b) + of "==", "!=": + result = evalCondCmp(c, s, name == "==") of "par": - # a parenthesised grouping such as `(defined(a) or defined(b))`: evaluate - # the inner expression. Without this, `par` fell through to the `else` - # branch below and evaluated to `true`, which silently inverted conditions - # like `not (defined(macosx) or defined(bsd))` and dropped real imports - # (e.g. `cpuinfo`'s conditional `import std/posix`). - result = evalCondExpr(c, s) - var depth = 1 - while depth > 0: - let n = next(s) - if n.kind == ParLe: inc depth - elif n.kind == ParRi: dec depth - elif n.kind == EofToken: return + # a parenthesised grouping such as `(defined(a) or defined(b))`. + result = readCond(c, s) else: - skipSubtree(s, t) - result = true + result = cvUnknown + # Drain whatever remains until the matching ParRi. + var depth = 1 + while depth > 0: + let n = next(s) + if n.kind == ParLe: inc depth + elif n.kind == ParRi: dec depth + elif n.kind == EofToken: return else: - result = true + result = cvUnknown -proc whenMarkerHolds(c: DepContext; s: var Stream): bool = +proc whenMarkerHolds(c: DepContext; s: var Stream): CondVal = ## Caller has just consumed the `(when` ParLe. Read children until the - ## matching `)`, AND-ing each evaluated condition. - result = true + ## matching `)`, AND-ing each evaluated condition. Returns the tri-state + ## result; callers keep the dependency unless it is provably `cvFalse`. + result = cvTrue while true: - # peek by reading; if it's ParRi, we're done let t = next(s) if t.kind == ParRi: return if t.kind == EofToken: return - if t.kind == ParLe: - # Re-feed by manually evaluating the subtree starting at `t`. - # evalCondExpr expects to read its own opener, so handle it directly. - let tag = pool.tags[t.tagId] - case tag - of "call", "cmd", "callstrlit", "infix", "prefix": - let head = next(s) - var name = "" - if head.kind == Ident: name = pool.strings[head.litId] - var ok = true - case name - of "defined": - let arg = next(s) - var sym = "" - if arg.kind == Ident: sym = pool.strings[arg.litId] - ok = sym.len > 0 and isDefined(c.config, sym) - of "not": - ok = not evalCondExpr(c, s) - of "and": - ok = evalCondExpr(c, s) - if ok: ok = evalCondExpr(c, s) - of "or": - ok = evalCondExpr(c, s) - if not ok: ok = evalCondExpr(c, s) - else: - ok = true - # finish the subtree - var depth = 1 - while depth > 0: - let n = next(s) - if n.kind == ParLe: inc depth - elif n.kind == ParRi: dec depth - elif n.kind == EofToken: return - if not ok: result = false - of "not", "and", "or": - # Re-emit a synthetic dispatch: rewrap by descending. - var ok = true - case tag - of "not": - ok = not evalCondExpr(c, s) - of "and": - ok = evalCondExpr(c, s) - if ok: ok = evalCondExpr(c, s) - of "or": - ok = evalCondExpr(c, s) - if not ok: ok = evalCondExpr(c, s) - else: discard - var depth = 1 - while depth > 0: - let n = next(s) - if n.kind == ParLe: inc depth - elif n.kind == ParRi: dec depth - elif n.kind == EofToken: return - if not ok: result = false - else: - # Unknown — treat as true and skip. - skipSubtree(s, t) - elif t.kind == Ident: - if not evalCondIdent(c, pool.strings[t.litId]): result = false - # a true / unknown ident keeps the current result + result = condAnd(result, evalCondExpr(c, s, t)) proc parseImportPath(s: var Stream; t: var PackedToken): seq[string] = ## Parse an import path expression and return the list of module paths it @@ -543,8 +552,12 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) = var live = true if t.kind == ParLe and pool.tags[t.tagId] == "when": # whenMarkerHolds consumes everything up to and including the - # closing `)` of the `(when ...)` subtree. - live = whenMarkerHolds(c, s) + # closing `)` of the `(when ...)` subtree. Drop the import only when + # the condition is PROVABLY false; a `cvUnknown` condition (e.g. an + # `else:` branch guarded by `not `, as in + # `when tryImport x: ... else: import x`) keeps the dependency so the + # static graph never misses a real import. + live = whenMarkerHolds(c, s) != cvFalse t = next(s) if not live: # Drain the rest of this import/include node. @@ -568,15 +581,15 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) = # be treated as modules. Both still create a real dependency on `m`. for importPath in parseImportPath(s, t): if importPath.len > 0: - processImport(c, importPath, current) + processImport(c, importPath, current, pair.nimFile) else: while t.kind != ParRi and t.kind != EofToken: for importPath in parseImportPath(s, t): if importPath.len > 0: if tag == "include": - processInclude(c, importPath, current) + processInclude(c, importPath, current, pair.nimFile) else: - processImport(c, importPath, current) + processImport(c, importPath, current, pair.nimFile) # Drain any remaining tokens of this node (e.g. the symbol list of a # `fromimport`), up to and including the node's closing ')'. var depth = 1 @@ -694,15 +707,14 @@ proc computeForwardedArgs(c: DepContext): seq[string] = # then abort builds the whole-program compilation accepts. Forward the # real project so children filter diagnostics identically. result.add "--icproject:" & c.config.projectFull.string - # Precompiled config: serialise the driver's config once and have every - # child replay it instead of re-parsing the `nim.cfg` chain and re-running - # `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig` - # restores the old per-child config parsing (for bisecting a suspected - # config-replay divergence without clearing caches). - if not isDefined(c.config, "icNoPreparsedConfig"): - let cfgArtifact = nimcache / "ic_config.cfg.nif" - writeIcConfig(c.config, cfgArtifact) - result.add "--icPreparsedConfig:" & cfgArtifact + # Precompiled config: every child replays the one artifact produced (in a + # separate `nim icconfig` process) and already replayed by the driver itself — + # see `icconfig.ensureIcConfig`, run before the driver's own `loadConfigs`. So + # `nim ic` is always governed by this single artifact, for speed and so the + # driver and its children agree by construction. Forward the path the driver + # replayed (`conf.icPreparsedConfig`); `commandIc` has already guaranteed it + # exists, else it bailed. + result.add "--icPreparsedConfig:" & c.config.icPreparsedConfig proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): string = ## Frontend build file: the nifler (parse) and `nim m` (sem) rules only. The @@ -1029,6 +1041,12 @@ proc commandIc*(conf: ConfigRef) = rawMessage(conf, errGenerated, "nifler tool not found. Install nimony or add nifler to PATH.") return + # Resolve the `.nim` source first, exactly like `wantMainModule`. Without + # this, an extensionless project arg (`nim ic path/to/foo`) resolves to a + # same-named sibling that already exists — e.g. the ELF a prior `nim c` + # left behind — and nifler chokes on the binary (`invalid token \127`, + # ELF magic). `addFileExt` only appends when there is no extension. + conf.projectFull = addFileExt(conf.projectFull, NimExt) let projectFile = conf.projectFull.string if not fileExists(projectFile): rawMessage(conf, errGenerated, "project file not found: " & projectFile) @@ -1118,6 +1136,15 @@ proc commandIc*(conf: ConfigRef) = # from its importer — and rerun; nifmake's mtime pruning keeps completed # work. A round that discovers nothing new but still fails is a real error. let forwardedArgs = computeForwardedArgs(c) + # The precompiled config drives every `nim m`/`nim nifc` child and the driver + # itself (`ensureIcConfig` produced it and `loadConfigs` replayed it). If it + # is not on disk something went wrong producing it — children would each + # silently fall back to re-parsing the whole config chain — so refuse to + # continue without it. + if conf.icPreparsedConfig.len == 0 or not fileExists(conf.icPreparsedConfig): + rawMessage(conf, errGenerated, + "precompiled config missing: " & conf.icPreparsedConfig) + return let nifmake = findNifmake() # Build the per-module rules concurrently: nifmake fans out all commands at # each DAG depth via execProcesses (defaults to all cores). Cold builds are @@ -1164,8 +1191,9 @@ proc commandIc*(conf: ConfigRef) = let newNode = Node(files: @[pair], id: c.nodes.len) if c.systemNodeId >= 0: newNode.deps.add c.systemNodeId - for impId in c.implicitNodeIds: - if impId != newNode.id: newNode.deps.add impId + if getsImplicitImports(c, pair.nimFile): + for impId in c.implicitNodeIds: + if impId != newNode.id: newNode.deps.add impId c.processedModules[pair.modname] = newNode.id c.nodes.add newNode idx = newNode.id diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim index 9eea4ee499..e31b2be82a 100644 --- a/compiler/ic/replayer.nim +++ b/compiler/ic/replayer.nim @@ -19,8 +19,11 @@ import std/tables when defined(nimPreviewSlimSystem): import std/assertions -proc replayStateChanges*(module: PSym; g: ModuleGraph) = - let list = module.ast +proc replayStateChanges*(module: PSym; g: ModuleGraph; list: PNode) = + ## `list` is an `nkStmtList` of `nkReplayAction` nodes (macro-cache puts/incs/ + ## adds/incls and a few pragmas) recorded for `module`. Under the NIF backend a + ## loaded module's `ast` is never reconstructed, so the caller passes the replay + ## actions it parsed out of the module's NIF directly. assert list != nil assert list.kind == nkStmtList for n in list: @@ -64,8 +67,9 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) = g.cacheTables[destKey] = initBTree[string, PNode]() if not contains(g.cacheTables[destKey], key): g.cacheTables[destKey].add(key, val) - else: - internalError(g.config, n.info, "key already exists: " & key) + # else: the same key was already replayed. Under IC the import closure is + # replayed (direct module + transitive deps), so the same registration can + # legitimately be reached twice; re-applying it is a no-op, not an error. of "incl": let destKey = n[1].strVal let val = n[2] diff --git a/compiler/icconfig.nim b/compiler/icconfig.nim index 653fb2dfd6..0250ee0583 100644 --- a/compiler/icconfig.nim +++ b/compiler/icconfig.nim @@ -31,23 +31,39 @@ ## `--path` arguments; replaying their raw, config-dir-relative arguments here ## would misresolve. -import options, commands, lineinfos -import std/[algorithm, os, sets] +import options, commands, lineinfos, pathutils, msgs +import std/[algorithm, os, sets, osproc, times, streams, syncio] import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse] const - IcConfigVersion* = "1" + IcConfigVersion* = "2" ## Artifact format version. Bump on any layout change here so a child built ## by an older compiler rejects a stale artifact and falls back to normal ## config loading instead of replaying a format it cannot parse. proc writeIcConfig*(conf: ConfigRef; outfile: string) = - ## Serialise the config-file switches recorded during `loadConfigs` plus the - ## resolved `cppDefines` set into the artifact at `outfile`. - var b = nifbuilder.open(outfile) + ## Serialise the resolved config (the config-file switches recorded during + ## `loadConfigs`, the resolved `cppDefines`/`searchPaths`, the nimcache dir, and + ## the list of config *source* files for staleness detection) into `outfile`. + ## `OnlyIfChanged`: when the content is byte-identical to what is already on + ## disk the file is left untouched so its mtime does not advance — otherwise + ## every `nim ic` run would re-fire the whole nifmake graph (see `nifler`'s + ## `produceConfig`, whose model this mirrors). + var b = nifbuilder.open(outfile, writeMode = OnlyIfChanged) b.withTree "stmts": b.withTree "meta": b.addStrLit IcConfigVersion + b.withTree "sources": + # Every config file read while loading (nim.cfg chain + config.nims), so a + # later run can decide via mtimes whether this artifact is still current + # (see `sourcesChanged`). + for f in conf.configFiles: + b.addStrLit f.string + b.withTree "nimcache": + # Resolved build nimcache. Recorded (unlike the path-search switches) so the + # driver, which replays this artifact instead of parsing `nim.cfg`, still + # learns a `--nimcache:` set inside `nim.cfg` and builds in the right place. + b.addStrLit conf.nimcacheDir.string b.withTree "cppdefines": # HashSet iteration order is unspecified; sort so the artifact is # byte-stable across runs (nifmake keys rebuilds off content changes). @@ -55,6 +71,15 @@ proc writeIcConfig*(conf: ConfigRef; outfile: string) = for d in conf.cppDefines: defs.add d sort defs for d in defs: b.addStrLit d + b.withTree "searchpaths": + # The resolved (absolute) search paths. Path-search *switches* are skipped + # below because their raw arguments are config-dir-relative; the net effect + # lives here instead, so a replayer with no `--path` command-line arguments + # (the `nim ic` driver itself) still resolves imports. `nim m`/`nim nifc` + # children also receive these as forwarded `--path` args; the dedup on + # replay makes the overlap harmless. + for p in conf.searchPaths: + b.addStrLit p.string b.withTree "switches": for sw in conf.icConfigSwitches: b.addTree "sw" @@ -74,7 +99,10 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool = let stmtsTag = tags.registerTag("stmts") metaTag = tags.registerTag("meta") + sourcesTag = tags.registerTag("sources") + nimcacheTag = tags.registerTag("nimcache") cppTag = tags.registerTag("cppdefines") + pathsTag = tags.registerTag("searchpaths") switchesTag = tags.registerTag("switches") swTag = tags.registerTag("sw") var buf = parseFromFile(infile, 1000, pool, tags) @@ -95,6 +123,22 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool = inc c else: skip c + elif c.cursorTagId == nimcacheTag: + c.loopInto: + if c.kind == StrLit: + let nc = strVal(c) + # Only when nimcache was not already pinned on the command line: a + # `--nimcache:` argument the driver/child was launched with must win + # over whatever `nim.cfg` recorded into the artifact. + if nc.len > 0 and conf.nimcacheDir.isEmpty: + conf.nimcacheDir = AbsoluteDir(nc) + inc c + else: + skip c + elif c.cursorTagId == sourcesTag: + # Replay does not need the source list; it exists only for + # `sourcesChanged`. Skip the whole section. + skip c elif c.cursorTagId == cppTag: c.loopInto: if c.kind == StrLit: @@ -102,6 +146,17 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool = inc c else: skip c + elif c.cursorTagId == pathsTag: + c.loopInto: + if c.kind == StrLit: + # Append preserving the serialised order (which already reflects the + # driver's addPath insert-at-front sequence), deduping against any + # path a child already received via a forwarded `--path` argument. + let d = AbsoluteDir(strVal(c)) + if not conf.searchPaths.contains(d): conf.searchPaths.add d + inc c + else: + skip c elif c.cursorTagId == switchesTag: c.loopInto: if c.kind == TagLit and c.cursorTagId == swTag: @@ -125,3 +180,104 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool = skip c endRead(c) result = sawMeta and version == IcConfigVersion + +proc sourcesChanged*(configFile: string): bool = + ## True when the precompiled config at `configFile` is missing, malformed, + ## written by an incompatible version, or any recorded config *source* file is + ## newer than it (or has vanished) — i.e. the artifact must be regenerated. + ## Mirrors nifler's `sourcesChanged`: the source list lives inside the artifact + ## so this needs no out-of-band knowledge of which `nim.cfg`s were read. + if not fileExists(configFile): return true + let modtime = getLastModificationTime(configFile) + var pool = newPool() + var tags = newTagPool() + let + stmtsTag = tags.registerTag("stmts") + metaTag = tags.registerTag("meta") + sourcesTag = tags.registerTag("sources") + var buf = parseFromFile(configFile, 1000, pool, tags) + var c = beginRead(buf) + if c.kind != TagLit or c.cursorTagId != stmtsTag: + endRead(c) + return true + var version = "" + var depsChanged = false + c.loopInto: + if c.kind == TagLit and c.cursorTagId == metaTag: + c.loopInto: + if c.kind == StrLit: + version = strVal(c) + inc c + else: + skip c + elif c.kind == TagLit and c.cursorTagId == sourcesTag: + c.loopInto: + if c.kind == StrLit: + let dep = strVal(c) + if not fileExists(dep) or getLastModificationTime(dep) >= modtime: + depsChanged = true + inc c + else: + skip c + else: + skip c + endRead(c) + result = depsChanged or version != IcConfigVersion + +proc produceIcConfig*(conf: ConfigRef) = + ## The `cmdIcConfig` command. By the time it runs, the normal pipeline has + ## already fully parsed the `nim.cfg` chain and run `config.nims`, so the + ## resolved config is sitting in `conf`; just serialise it to `--o`. + let outPath = conf.icConfigOut + if outPath.len == 0: + rawMessage(conf, errGenerated, "icconfig: missing output path (--icConfigOut)") + return + createDir(parentDir(outPath)) + writeIcConfig(conf, outPath) + +proc ensureIcConfig*(conf: ConfigRef) = + ## Driver-side (`cmdIc`). Make sure an up-to-date precompiled config exists, + ## (re)producing it in a *separate* process when missing or stale, then point + ## `conf.icPreparsedConfig` at it so the driver replays the very same config its + ## `nim m`/`nim nifc` children will — perfect speed (config parsed at most once, + ## skipped entirely when nothing changed) and consistency (one producer, every + ## process replays its output). The artifact lives in the nimcache derived from + ## the command line (pre-config-parse), which is the one the children are told; + ## a `--nimcache:` set inside `nim.cfg` is recovered from the artifact itself. + let cacheDir = getNimcacheDir(conf).string + # Start from a clean cache when the on-disk NIF format stamp is absent or stale + # (see `icFormatVersion`). This must happen HERE, before the config artifact is + # produced — `commandIc` performs the same check later, but by then the artifact + # would already live in the cache and the wipe would delete it. + createDir(cacheDir) + let versionFile = cacheDir / "ic.version" + let stamp = if fileExists(versionFile): readFile(versionFile) else: "" + if stamp != icFormatVersion: + removeDir(cacheDir) + createDir(cacheDir) + writeFile(versionFile, icFormatVersion) + let outPath = cacheDir / "ic_config.cfg.nif" + if not fileExists(outPath) or sourcesChanged(outPath): + createDir(cacheDir) + # Re-invoke ourselves as the config producer: reuse this process's command + # line, dropping the command argument (`ic`) in favour of `icconfig` and the + # explicit output path, both BEFORE the project file (anything after the + # project is swallowed into `config.arguments` by `cmdLineRest`). The + # producer re-reads `nim.cfg` itself. + var pargs = @["icconfig", "--icConfigOut:" & outPath] + var droppedCmd = false + for a in commandLineParams(): + if not droppedCmd and a.len > 0 and a[0] != '-': + droppedCmd = true # drop the original command token (`ic`) + else: + pargs.add a + let p = startProcess(getAppFilename(), args = pargs, + options = {poStdErrToStdOut}) + let outp = p.outputStream.readAll() + let code = p.waitForExit() + p.close() + if code != 0 or not fileExists(outPath): + rawMessage(conf, errGenerated, + "failed to produce precompiled config (exit code " & $code & "):\n" & outp) + return + conf.icPreparsedConfig = outPath diff --git a/compiler/main.nim b/compiler/main.nim index bf7eb2b675..9052287a1f 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -29,6 +29,7 @@ when defined(nimPreviewSlimSystem): import ../dist/checksums/src/checksums/sha1 import pipelines +from icconfig import produceIcConfig when not defined(nimKochBootstrap): import nifbackend @@ -439,6 +440,11 @@ proc mainCommand*(graph: ModuleGraph) = commandIc(conf) else: rawMessage(conf, errGenerated, "nim deps not available in bootstrap build") + of cmdIcConfig: + # Produce the precompiled config artifact for `nim ic` (config already + # parsed by the normal pipeline); a separate process spawned by the driver. + wantMainModule(conf) + produceIcConfig(conf) of cmdParse: wantMainModule(conf) discard parseFile(conf.projectMainIdx, cache, conf) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index e145214bcb..323ff8fafd 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -146,6 +146,11 @@ type cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API; IC: implemented cacheCounters*: Table[string, BiggestInt] # IC: implemented cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented + transitiveReplayActions*: seq[PNode] # macro-cache replay actions collected from + # the transitive import closure of a NIF-loaded module (loadTransitiveHooks); + # the caller (pipelines) replays them so a dependency's macrocache state — e.g. + # nim-serialization's flavor registration — reaches a module that imports it + # only indirectly. Drained per moduleFromNifFile call. passes*: seq[TPass] pipelinePass*: PipelinePass onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} @@ -944,6 +949,14 @@ when not defined(nimKochBootstrap): if not g.hookClosure.containsOrIncl(fileIdx.int): let precomp = loadNifModule(ast.program, suffix, interf, interfHidden, {}) registerLoadedHooks(g, precomp.logOps) + # Collect the dependency's macro-cache replay actions (put/inc/add/incl) + # so the importer being compiled also sees macrocache state registered + # by a transitively-imported module. Pragma replay actions are a backend + # concern and are intentionally not collected here. + for n in precomp.topLevel: + if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and + n[0].strVal in ["put", "inc", "add", "incl"]: + g.transitiveReplayActions.add n for d in precomp.deps: stack.add d proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym = @@ -1009,6 +1022,15 @@ when not defined(nimKochBootstrap): if ms != nil: strTableAdd(g.ifaces[fileIdx.int].interf, ms) + # Rebuild `procInstCache` from this module's generic-instance OFFERS so a + # consumer's `genericCacheGet` finds the instance and SKIPS re-running + # `instantiateBody` in its own module scope (which lacks symbols visible only + # at the generic's definition site — see ast2nif's `(offer …)`). + for off in result.genericOffers: + g.procInstCache.mgetOrPut(off.generic.itemId, @[]).add PInstantiation( + sym: off.inst, concreteTypes: off.concreteTypes, + genericParamsCount: off.genericParamsCount, compilesId: 0) + # Mark module as cached g.cachedMods.incl fileIdx.int g.hookClosure.incl fileIdx.int @@ -1019,6 +1041,11 @@ when not defined(nimKochBootstrap): case x.kind of ConverterEntry: g.ifaces[fileIdx.int].converters.add x.sym + of PureEnumEntry: + # rebuild the pure-enum list (source path: `addPureEnum`) so importers can + # offer this loaded `{.pure.}` enum's fields as the restricted pure-enum + # fallback (`importPureEnumFields`). + g.ifaces[fileIdx.int].pureEnums.add x.sym of MethodEntry: discard "dispatch buckets already rebuilt by registerLoadedHooks" of GenericInstEntry: @@ -1063,7 +1090,16 @@ proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym = proc belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool = ## Check if symbol belongs to the 'stdlib' package. - sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId + # Compare the package *name* (an interned ident), not the package symbol's + # `.id`. Under per-module IC (`nim m`) the system module is loaded from a NIF + # in a process that does not compile it from source, so its package symbol is + # reconstructed with a fresh `.id` that no longer matches the freshly-interned + # package of a stdlib module compiled standalone here — making the old id + # comparison wrongly report `false` and inject `--import`ed modules into the + # stdlib. Both are canonically named `stdlib` (lib/stdlib.nimble); in a normal + # `nim c` build (system compiled from source) the ids match too, so this is a + # no-op there. + sym.getPackageSymbol.name.id == graph.systemModule.getPackageSymbol.name.id proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): SuggestFileSymbolDatabase = result = graph.suggestSymbols.getOrDefault(fileIdx, newSuggestFileSymbolDatabase(fileIdx, optIdeExceptionInlayHints in graph.config.globalOptions)) diff --git a/compiler/nim.nim b/compiler/nim.nim index fd33aca040..0d2a0a95a4 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -29,6 +29,7 @@ import pathutils, modulegraphs from ast2nif import registerNifAstTags +from icconfig import ensureIcConfig from std/browsers import openDefaultBrowser from nodejs import findNodeJs @@ -114,6 +115,14 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = self.processCmdLineAndProjectPath(conf) + # `nim ic` driver: ensure the precompiled config exists (produced by a separate + # `nim icconfig` process, skipped when nothing changed) BEFORE config loading, + # so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the + # driver runs on the exact same config its children will. See icconfig.nim. + when not defined(nimKochBootstrap): + if conf.cmd == cmdIc: + ensureIcConfig(conf) + var graph = newModuleGraph(cache, conf) if not self.loadConfigsAndProcessCmdLine(cache, conf, graph): return diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 9c5e4c6d07..fc5f4e3656 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -246,10 +246,14 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = setDefaultLibpath(conf) - # `nim ic` children replay the precompiled config the driver recorded once, - # instead of re-reading the `nim.cfg` chain and re-running `config.nims` in the - # VM. A missing/format-incompatible artifact returns false: fall through to - # normal config loading so an older child or a deleted cache still works. + # The `nim ic` driver and its `nim m`/`nim nifc` children replay the precompiled + # config (produced once by a separate `nim icconfig` process — see + # `icconfig.ensureIcConfig`, which sets `icPreparsedConfig` for the driver + # before this runs; the children get it as a forwarded `--icPreparsedConfig` + # argument) instead of re-reading the `nim.cfg` chain and re-running + # `config.nims` in the VM. A missing/format-incompatible artifact returns false: + # fall through to a normal parse (this is also the path the `nim icconfig` + # producer itself takes, since it runs with no `icPreparsedConfig`). if conf.icPreparsedConfig.len > 0 and applyIcConfig(conf, conf.icPreparsedConfig): return template readConfigFile(path) = diff --git a/compiler/options.nim b/compiler/options.nim index 012f633946..8559caf0e3 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -200,6 +200,7 @@ type cmdCompileToNif cmdNifC # generate C code from NIF files cmdIc # generate .build.nif for nifmake + cmdIcConfig # `nim ic`'s precompiled-config producer (writes ic_config.cfg.nif) const cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, @@ -418,12 +419,16 @@ type # module's package the "main package" and unfilter # foreign-package diagnostics; the real project # restores whole-program filtering semantics. - icPreparsedConfig*: string # under `nim m`/`nim nifc`: path of the precompiled - # config artifact written once by the `nim ic` driver. + icPreparsedConfig*: string # under the `nim ic` driver and its `nim m`/`nim nifc` + # children: path of the precompiled config artifact. # When set, `loadConfigs` replays the recorded # config-file switches from it instead of re-reading # the `nim.cfg` chain and re-running `config.nims` - # (which the VM makes expensive) per subprocess. + # (which the VM makes expensive) per process. The + # artifact itself is produced by a separate + # `nim icconfig` process (see `cmdIcConfig`). + icConfigOut*: string # under `nim icconfig`: the path to write the + # precompiled config artifact to (set via `--o`). icConfigSwitches*: seq[tuple[switch, arg: string]] # the config-file (`passPP`) switches applied while # loading config, in order. Recorded by every nim diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 7c9ce97ff3..3701d0cfb2 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -269,23 +269,94 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator # the union; intra-group entries are filtered by the writer. var implDeps: seq[int] = @[] for id in graph.icImplDeps: implDeps.add id + # Generic-instance OFFERS: every instance THIS module created, so a + # consumer reuses it rather than re-instantiating in its own scope (which + # cannot see symbols visible only at the generic's definition site — e.g. + # a distinct type's `==`). See ast2nif.writeNifModule / moduleFromNifFile. + var genericOffers: seq[tuple[generic, inst: PSym; + concreteTypes: seq[PType]; genericParamsCount: int]] = @[] + for genItemId, instList in graph.procInstCache: + for inst in instList: + if inst.sym != nil and inst.sym.itemId.module == module.position and + inst.sym.instantiatedFrom != nil and inst.compilesId == 0: + # `concreteTypes` is pre-sized to `paramsLen+gp.len`; a tail slot can + # stay nil (e.g. fewer materialized params than `paramsLen`). Such an + # offer can't be serialized — skip it (the consumer re-instantiates, + # the prior behaviour) rather than emit a nil type reference. + var hasNil = false + for ct in inst.concreteTypes: + if ct == nil: hasNil = true; break + if not hasNil: + genericOffers.add (inst.sym.instantiatedFrom, inst.sym, + inst.concreteTypes, inst.genericParamsCount) + # The module's REAL resolved direct imports (incl. macro/template-generated + # ones with no surviving syntactic node). Passed to writeNifModule so the + # NIF `deps` section is complete (the backend closure walk needs it), and + # reused below for the `.s.deps` sidecar (frontend graph re-derivation). + let resolvedImportDeps = graph.importDeps.getOrDefault(module.position.FileIndex, @[]) writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, - replayActions, implDeps, reexportedModuleSyms(graph, module)) + replayActions, implDeps, reexportedModuleSyms(graph, module), + genericOffers, resolvedImportDeps) # The module's REAL direct imports (incl. macro-generated) for `nim ic`'s # graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep. var semDepPaths: seq[string] = @[] - for f in graph.importDeps.getOrDefault(module.position.FileIndex, @[]): + for f in resolvedImportDeps: semDepPaths.add toFullPath(graph.config, f) writeSemDeps(graph.config, module.position.int32, semDepPaths) result = true +proc loadedDefSym(defs: PNode): PSym = + ## The defined symbol of a let/var entry as it loads back from a NIF: the + ## section child is a bare `nkSym` (the `(sd …)` reference), but be defensive + ## about the from-source shapes too (`nkIdentDefs`, a pragma-wrapped name). + case defs.kind + of nkSym: result = defs.sym + of nkPragmaExpr: + result = if defs.len > 0: loadedDefSym(defs[0]) else: nil + of nkIdentDefs, nkConstDef: + result = if defs.len > 0: loadedDefSym(defs[0]) else: nil + else: result = nil + +proc initLoadedCompileTimeGlobals(graph: ModuleGraph; module: PSym; topLevel: PNode) = + ## Eagerly initialize the compile-time globals (`let/var {.compileTime.}`) of a + ## module restored from a NIF. In a normal sem these VM slots are filled by + ## `setupCompileTimeVar` (semstmts) as the section is semchecked; a NIF-loaded + ## module is never semchecked, so without this a macro or compile-time proc that + ## reads such a global finds a nil slot. The lazy `vmgen.genGlobalInit` fallback + ## is order-fragile across proc boundaries (it emits the init at the first + ## VM-gen'd reference, which need not be the first one executed), so the init has + ## to happen here, once, before any of the module's code can run. The symbol's + ## own `ast` is the `nkIdentDefs` (initializer included); re-wrap it in a section + ## exactly as semstmts does and hand it to the same evaluator. + if topLevel == nil: return + let idgen = idGeneratorFromModule(module) + for stmt in topLevel: + if stmt.kind notin {nkLetSection, nkVarSection}: continue + for defs in stmt: + let s = loadedDefSym(defs) + if s != nil and s.kind in {skLet, skVar} and + {sfCompileTime, sfGlobal} <= s.flags and + s.ast != nil and s.ast.kind == nkIdentDefs: + var sect = newNodeI(stmt.kind, s.info) + sect.add s.ast + setupCompileTimeVar(module, idgen, graph, sect) + proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags; fromModule: PSym = nil): PSym = var flags = flags if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule result = graph.getModule(fileIdx) template processModuleAux(moduleStatus) = + when defined(icDbg): + block: + let dbgf = open("/tmp/defdbg.txt", fmAppend) + dbgf.writeLine toFullPath(graph.config, fileIdx) & + " nimStackTraceOverride=" & $isDefined(graph.config, "nimStackTraceOverride") & + " nimscript=" & $isDefined(graph.config, "nimscript") & + " optCompress=" & $(optCompress in graph.config.globalOptions) & + " cmd=" & $graph.config.cmd + dbgf.close() onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) var s: PLLStream = nil if sfMainModule in flags: @@ -333,9 +404,33 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF if sfSystemModule in flags: graph.systemModule = result partialInitModule(result, graph, fileIdx, AbsoluteFile(toFullPath(graph.config, fileIdx))) - # Replay state changes from the loaded NIF module - if result.ast != nil: - replayStateChanges(result, graph) + # Replay the module's recorded state changes: macro-cache operations + # (std/macrocache puts/incs/adds/incls) plus a few pragmas. The loader + # parsed them into `precomp.topLevel` (mixed with other top-level nodes), + # so filter to the replay actions. A loaded module's `ast` is never + # rebuilt, so this used to be skipped (`result.ast == nil`) and a + # NIF-loaded module's macro cache was lost — e.g. nim-serialization's + # flavor registration became invisible to dependents (`DefaultFlavor: + # automatic serialization is not enabled`). + var replayList = newNodeI(nkStmtList, result.info) + for n in precomp.topLevel: + # Only macro-cache ops (put/inc/add/incl). The pragma replay actions + # (compile/link/passc/hint/...) are a backend/link concern handled by + # the nifc closure, and re-emitting a loaded module's hints/warnings on + # every import would be wrong — so they are deliberately skipped here. + if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and + n[0].strVal in ["put", "inc", "add", "incl"]: + replayList.add n + # Plus the macro-cache actions of the module's transitive import closure + # (collected by the moduleFromNifFile call above via loadTransitiveHooks), + # so a flavor/type registered in an indirectly-imported module is visible. + for n in graph.transitiveReplayActions: replayList.add n + graph.transitiveReplayActions.setLen 0 + if replayList.len > 0: + replayStateChanges(result, graph, replayList) + # Fill the VM slots of the module's `{.compileTime.}` globals now (sem + # would have, but a NIF-loaded module is never semchecked). + initLoadedCompileTimeGlobals(graph, result, precomp.topLevel) return result # Return early, don't process from source let path = toFullPath(graph.config, fileIdx) let filename = AbsoluteFile path diff --git a/compiler/semcall.nim b/compiler/semcall.nim index a27531c186..38b88ee7d2 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -983,7 +983,12 @@ proc explicitGenericSym(c: PContext, n: PNode, s: PSym, errors: var CandidateErr diagnostics: m.diagnostics)) return nil var newInst = generateInstance(c, s, m.bindings, n.info) - newInst.typ.excl tfUnresolved + # `generateInstance` may return an instance REUSED from another module's NIF + # `(offer …)` — its type is Sealed (immutable). Such an instance is already + # fully resolved (`tfUnresolved` cleared at its original instantiation), so the + # `excl` is a no-op; skip it rather than assert on a Sealed-type mutation. + if newInst.typ.state != Sealed: + newInst.typ.excl tfUnresolved let info = getCallLineInfo(n) markUsed(c, info, s, isGenericInstance = false) onUse(info, s, isGenericInstance = false) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index c057b1f23d..a15bc0ef5a 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -410,6 +410,13 @@ proc addConverterDef*(c: PContext, conv: PSym) = proc addPureEnum*(c: PContext, e: PSym) = assert e != nil add(c.graph.ifaces[c.module.position].pureEnums, e) + # record for IC: a NIF-loaded module rebuilds `Iface.pureEnums` from these log + # entries (moduleFromNifFile); without it a loaded module's pure enums were + # invisible to importers, so `importPureEnumFields` never offered their fields + # and unqualified pure-enum values stopped resolving. (Same pattern as + # `addConverterDef`.) + c.graph.opsLog.add LogEntry(kind: PureEnumEntry, module: c.module.position, + key: "", sym: e) proc addPattern*(c: PContext, p: PSym) = assert p != nil diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 27bf4e4319..3bbbc1b97e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1905,6 +1905,22 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = n.typ = n.typ.elementType result.add(n) +proc markResultVarIsPtr(c: PContext, x: PNode) {.inline.} = + ## Set `tfVarIsPtr` on the (result) sym node's type. Under IC that type can be a + ## NIF-loaded (Sealed) and interned instance which must not be mutated in place + ## (it could corrupt other users of the shared type, and the assert forbids it): + ## give this result its own copy carrying the flag, exactly like a from-source + ## compile has a fresh result type here. + if tfVarIsPtr in x.typ.flags: return + if x.typ.state == Sealed: + let fresh = copyType(x.typ, c.idgen, x.typ.owner) + fresh.incl tfVarIsPtr + x.typ = fresh + if x.kind == nkSym and x.sym.state != Sealed: + x.sym.typ = fresh + else: + x.typ.incl tfVarIsPtr + proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = if le.kind == nkHiddenDeref: var x = le[0] @@ -1912,10 +1928,10 @@ proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = if x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView): n[0] = x # 'result[]' --> 'result' n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent) - x.typ.incl tfVarIsPtr + markResultVarIsPtr(c, x) #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info elif sfGlobal in x.sym.flags: - x.typ.incl tfVarIsPtr + markResultVarIsPtr(c, x) proc borrowCheck(c: PContext, n, le, ri: PNode) = const diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 67607b31ed..fdb089d7f4 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -149,6 +149,17 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi # as properties are accessed and trigger lazy loading. backendEnsureMutable(t) + # Bare type-class keywords used as a typedesc without arguments (e.g. `array`, + # `range`, `distinct` passed to `signatureHash`) have no children, so the + # structural branches below would index a non-existent `elementType`. Hash them + # by kind (+ sym for an extra, stable distinction) — enough for a stable, + # distinct identity. (`seq`/`openArray`/`tuple` already fall through the empty + # `else` loop unharmed; this covers the branches that index `elementType`.) + if t.kind in {tyArray, tyRange, tyDistinct} and not t.hasElementType: + c &= char(t.kind) + if t.sym != nil: c.hashSym(t.sym) + return + case t.kind of tyGenericInvocation: for a in t.kids: diff --git a/compiler/vm.nim b/compiler/vm.nim index a2baa5ec3c..851ba1d9db 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1955,7 +1955,21 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if regs[rb].node.kind != nkSym: stackTrace(c, tos, pc, "node is not a symbol") else: - regs[ra].node.strVal = $sigHash(regs[rb].node.sym, c.config) + let shSym = regs[rb].node.sym + # When `signatureHash` is applied to a type (e.g. a `T: typedesc`/generic + # param), hash the *type* it denotes, not the parameter symbol. Hashing the + # symbol routes through `hashNonProc`, which mixes in `s.disamb` — a + # per-module instantiation counter. Under incremental compilation the + # registering module and a consuming module instantiate the surrounding + # generic separately, get different `disamb`s, and produce different + # hashes for the same type (nim-serialization's auto-serialization lookup + # missed because of this). Hashing the underlying type via `hashType` is + # type-identity based and stable across the NIF boundary. + let shTyp = shSym.typ + if shTyp != nil and shTyp.kind == tyTypeDesc and shTyp.hasElementType: + regs[ra].node.strVal = $hashType(shTyp.elementType, c.config) + else: + regs[ra].node.strVal = $sigHash(shSym, c.config) of opcSlurp: decodeB(rkNode) createStr regs[ra] diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 2b1ec37283..999b0756de 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1782,8 +1782,15 @@ proc genGlobalInit(c: PCtx; n: PNode; s: PSym) = # This is rather hard to support, due to the laziness of the VM code # generator. See tests/compile/tmacro2 for why this is necessary: # var decls{.compileTime.}: seq[NimNode] = @[] + # Load the slot's ADDRESS (not its value): the lazy initializer must REPLACE + # the null slot, which `opcWrDeref` only does for an `rkNodeAddr` target + # (`nAddr[] = n` for refs). With `opcLdGlobal` the slot value is loaded and for + # a ref-typed global that value is an `nkNilLit` ("nil ref"); writing through it + # hits the VM's nil-deref guard ("attempt to access a nil address"). This path + # is reached for compile-time globals whose defining module is restored from a + # NIF under `nim ic` (so `setupCompileTimeVar` never ran to eagerly init them). let dest = c.getTemp(s.typ) - c.gABx(n, opcLdGlobal, dest, s.position) + c.gABx(n, opcLdGlobalAddr, dest, s.position) if s.astdef != nil: let tmp = c.genx(s.astdef) c.genAdditionalCopy(n, opcWrDeref, dest, 0, tmp) diff --git a/koch.nim b/koch.nim index 4127a86e25..809ddee868 100644 --- a/koch.nim +++ b/koch.nim @@ -600,19 +600,38 @@ proc xtemp(cmd: string) = finally: copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe) -proc icTest(args: string) = - temp("") - let inp = os.parseCmdLine(args)[0] +proc runIcTestFile(inp: string) = + ## Compile a single `tests/ic` file with `nim ic`, once per `#!EDIT!#` fragment + ## (each fragment is the file's source after that incremental edit). Only checks + ## that `nim ic` exits 0 — the produced binary's output is not verified here. let content = readFile(inp) let nimExe = getAppDir() / "bin" / "nim_temp".exe - var i = 0 for fragment in content.split("#!EDIT!#"): let file = inp.replace(".nim", "_temp.nim") writeFile(file, fragment) var cmd = nimExe & " ic --hint:Conf:off --warnings:off " cmd.add quoteShell(file) exec(cmd) - inc i + +# The `tests/ic` files that `nim ic` must keep compiling. Multi-module tests rely +# on a sibling helper (`timp` -> `myimp`, `tcompiletimeglobal` -> `mctglobal`), +# which exercises the NIF import/load path the single-file tests do not. +const icSuite = ["thallo", "tconverter", "timp", "tmiscs", "tparseutils", + "tcompiletimeglobal", "tsighashstable", "tpureenum", "tgenericoffer"] + +proc icTest(args: string) = + temp("") + let parsed = os.parseCmdLine(args) + if parsed.len > 0 and parsed[0].len > 0: + # `koch ic `: run just that file. + runIcTestFile(parsed[0]) + else: + # `koch ic`: the full regression set we want to keep working — the test + # suite plus both self-host bootstraps (`bootic` and `bootic -d:release`). + for t in icSuite: + runIcTestFile("tests" / "ic" / (t & ".nim")) + bootic("", skipIntegrityCheck = false) + bootic("-d:release", skipIntegrityCheck = false) proc buildDrNim(args: string) = if not dirExists("dist/nimz3"): diff --git a/tests/ic/mctglobal.nim b/tests/ic/mctglobal.nim new file mode 100644 index 0000000000..8a84567554 --- /dev/null +++ b/tests/ic/mctglobal.nim @@ -0,0 +1,28 @@ +# Helper module for tcompiletimeglobal.nim (not a test itself; no `discard`). +# +# Exercises `{.compileTime.}` module-level globals across the NIF boundary: under +# `nim ic` this module is compiled to its own NIF and *loaded* (not semchecked) +# by the importer, so its compile-time globals must be eagerly initialized at +# load time. A macro that splices such a global into a `quote do:` otherwise +# reads a nil VM slot. + +import macros + +let injectedName {.compileTime.} = ident "ctgValue" + +proc genAssign*(): NimNode = + ## The order-fragile case: the CT global is read inside a proc that the macro + ## calls, not in the macro's own `quote`. The lazy VM init attaches to the + ## first vmgen'd reference, which need not be the first one executed. + result = quote do: + `injectedName` = 42 + +macro defineCtgValue*(): untyped = + result = newStmtList() + # Read the global via the helper proc FIRST, then via the macro's own quote, + # so the proc's reference executes before the macro's: only eager init at load + # time makes both see the initialized value. + let assign = genAssign() + result.add quote do: + var `injectedName`: int + result.add assign diff --git a/tests/ic/mgofmc.nim b/tests/ic/mgofmc.nim new file mode 100644 index 0000000000..678e1e12c9 --- /dev/null +++ b/tests/ic/mgofmc.nim @@ -0,0 +1,3 @@ +# Helper for tgenericoffer.nim: a distinct type whose `==` lives in THIS module. +type MultiCodec* = distinct int +proc `==`*(a, b: MultiCodec): bool = int(a) == int(b) diff --git a/tests/ic/mgofmh.nim b/tests/ic/mgofmh.nim new file mode 100644 index 0000000000..24cf5c6521 --- /dev/null +++ b/tests/ic/mgofmh.nim @@ -0,0 +1,10 @@ +# Helper for tgenericoffer.nim: imports mgofmc, builds a compile-time const +# Table[MultiCodec,int] and exposes a GENERIC proc whose (T-independent) body +# instantiates getOrDefault[MultiCodec] here, where mgofmc's `==` is visible. +import tables, mgofmc +proc buildTab(): Table[MultiCodec, int] = + result[MultiCodec(1)] = 100 + result[MultiCodec(2)] = 200 +const CodeHashes = buildTab() +proc digest*[T](x: T): int = + CodeHashes.getOrDefault(MultiCodec(2)) diff --git a/tests/ic/mgofpid.nim b/tests/ic/mgofpid.nim new file mode 100644 index 0000000000..9d13f8442d --- /dev/null +++ b/tests/ic/mgofpid.nim @@ -0,0 +1,6 @@ +# Helper for tgenericoffer.nim: imports mgofmh but NOT mgofmc, then calls the +# generic `digest`. Under IC this re-instantiates digest here, where mgofmc's +# `==` is NOT in scope — so getOrDefault[MultiCodec] must be REUSED from mgofmh's +# offer, not re-instantiated, or `==(MultiCodec)` resolution fails. +import mgofmh +proc callDigest*(): int = digest(5) diff --git a/tests/ic/mpureenum.nim b/tests/ic/mpureenum.nim new file mode 100644 index 0000000000..b39ceeb9ab --- /dev/null +++ b/tests/ic/mpureenum.nim @@ -0,0 +1,21 @@ +# Helper module for tpureenum.nim (not a test itself; no `discard`). +# +# Defines a `{.pure.}` enum whose field name (`Number`) collides with a distinct +# type of the same name. Under `nim ic` the pure enum is loaded from a NIF; its +# fields must NOT leak into the importer's unqualified scope (the source path +# keeps them out via `declarePureEnumField`). Before the fix, a loaded pure +# enum's fields were marked bare-importable and leaked into `interf`, so the +# field shadowed the distinct type and `uint64(x).Number` failed with +# "undeclared field 'Number' for type system.uint64" (the nim-json-serialization +# `JsonValueKind.Number` vs web3 `Number = distinct uint64` bug). + +type + Number* = distinct uint64 + + JsonValueKind* {.pure.} = enum + String, Number, Object, Array, Bool, Null + +proc val*(x: Number): uint64 = uint64(x) + +proc toNumber*(x: uint64): Number = + x.Number # must bind to the distinct TYPE `Number`, not the pure enum field diff --git a/tests/ic/msighashstable.nim b/tests/ic/msighashstable.nim new file mode 100644 index 0000000000..d5ce85d18a --- /dev/null +++ b/tests/ic/msighashstable.nim @@ -0,0 +1,48 @@ +# Helper module for tsighashstable.nim (not a test itself; no `discard`). +# +# Models nim-serialization's auto-serialization registry: a flavor records which +# types it auto-serializes in a `std/macrocache` keyed by `signatureHash(T)`, +# computed through a generic `{.compileTime.}` func. The registration happens +# here (at this module's compile time); the lookup happens in the importer. +# +# Under `nim ic` the two modules are compiled separately, so the generic +# `getSig[T]` is instantiated independently on each side. `signatureHash` must +# therefore hash the *type* `T` denotes, not the generic parameter symbol — the +# latter mixes in a per-module `disamb` counter and diverges across the NIF +# boundary, making the lookup miss. + +import std/[macrocache, macros, typetraits] + +type DefaultFlavor* = object + +macro calcSig*(T: typed): untyped = + doAssert(T.typeKind == ntyTypeDesc) + result = newLit(signatureHash(T)) + +func getSig*(F: type DefaultFlavor, T: distinct type): string {.compileTime.} = + calcSig(T) + +func getTable*(F: type DefaultFlavor): CacheTable {.compileTime.} = + CacheTable("nsrzStableTable" & typetraits.name(F)) + +func setAuto*(F: type DefaultFlavor, T: distinct type) {.compileTime.} = + let sig = F.getSig(T) + let table = F.getTable() + if not table.hasKey(sig): + table[sig] = newLit(1) + +func getAuto*(F: type DefaultFlavor, T: distinct type): bool {.compileTime.} = + let sig = F.getSig(T) + let table = F.getTable() + table.hasKey(sig) + +template autoCheck*(F: distinct type, T: distinct type, body) = + when not F.getAuto(T): + {.error: "auto serialization not enabled for `" & typetraits.name(T) & "`".} + else: + body + +static: + setAuto(DefaultFlavor, string) + setAuto(DefaultFlavor, SomeInteger) + setAuto(DefaultFlavor, seq) diff --git a/tests/ic/tcompiletimeglobal.nim b/tests/ic/tcompiletimeglobal.nim new file mode 100644 index 0000000000..044f972a31 --- /dev/null +++ b/tests/ic/tcompiletimeglobal.nim @@ -0,0 +1,16 @@ +discard """ +output: '''42''' +""" + +# Regression test: `{.compileTime.}` module-level globals of a NIF-loaded module +# must be initialized so macros/compile-time procs that splice them produce valid +# code. Before the eager-init fix this failed under `nim ic` with +# "attempt to access a nil address" / "illformed AST: break nil" / +# "undeclared identifier". `koch ic` only checks the compile succeeds; a nil +# global makes `defineCtgValue` emit `var : int` / ` = 42` and the +# compile fails. + +import mctglobal + +defineCtgValue() +echo ctgValue diff --git a/tests/ic/tgenericoffer.nim b/tests/ic/tgenericoffer.nim new file mode 100644 index 0000000000..dfd11278d4 --- /dev/null +++ b/tests/ic/tgenericoffer.nim @@ -0,0 +1,12 @@ +discard """ +output: '''200''' +""" + +# Regression test for the `(offer …)` generic-instance sharing across the NIF +# boundary. A generic loaded from a NIF whose body uses a type-bound op on a +# CONCRETE type (here getOrDefault on Table[MultiCodec,int]) must be reused, not +# re-instantiated in the consumer's scope which lacks the type's `==`. Before the +# fix this failed under `nim ic` with `hashcommon.nim type mismatch ... MultiCodec`. + +import mgofpid +echo callDigest() diff --git a/tests/ic/tpureenum.nim b/tests/ic/tpureenum.nim new file mode 100644 index 0000000000..36beb55622 --- /dev/null +++ b/tests/ic/tpureenum.nim @@ -0,0 +1,21 @@ +discard """ +output: '''5 +Number +Object''' +""" + +# Regression test: a `{.pure.}` enum loaded from a NIF must keep its fields out +# of the importer's unqualified scope (no leak), while still being reachable +# qualified AND via the restricted unambiguous-bare pure-enum fallback +# (`importPureEnumFields`, fed by `ifaces[].pureEnums` which the loader rebuilds +# from `PureEnumEntry` log ops). +# +# Before the fix the pure fields leaked into `interf` under `nim ic`, so +# `JsonValueKind.Number` shadowed the distinct `Number` type and the conversion +# below failed: "undeclared field 'Number' for type system.uint64". + +import mpureenum + +echo toNumber(5'u64).val # distinct conversion: field must NOT shadow type +echo JsonValueKind.Number # qualified pure-enum access still works +echo Object # unambiguous bare pure-enum field still resolves diff --git a/tests/ic/tsighashstable.nim b/tests/ic/tsighashstable.nim new file mode 100644 index 0000000000..11c20240d9 --- /dev/null +++ b/tests/ic/tsighashstable.nim @@ -0,0 +1,33 @@ +discard """ +output: '''ok string +ok int +ok seq''' +""" + +# Regression test: `signatureHash(T)` must be stable across the NIF boundary so +# that a macrocache keyed by it (nim-serialization's auto-serialization registry) +# can be populated in one module and queried from another under `nim ic`. +# +# Before the fix, `signatureHash` hashed the generic *parameter symbol* (whose +# `disamb` is a per-module instantiation counter) instead of the type it denotes. +# The registering module (msighashstable) and this importer instantiated the +# generic `getSig[T]` separately, got different `disamb`s, and the lookups for +# `string`/`SomeInteger` missed -> `{.error: auto serialization not enabled.}`. + +import msighashstable + +proc writeStr(F: type DefaultFlavor, v: string) = + autoCheck(F, string): + echo "ok string" + +proc writeInt[T: SomeInteger](F: type DefaultFlavor, v: T) = + autoCheck(F, SomeInteger): + echo "ok int" + +proc writeSeq[T](F: type DefaultFlavor, v: seq[T]) = + autoCheck(F, seq): + echo "ok seq" + +writeStr(DefaultFlavor, "hi") +writeInt(DefaultFlavor, 42) +writeSeq(DefaultFlavor, @[1, 2, 3])