mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-06-17 09:01:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

IC: precompiled configs and bugfixes (#25913)

Browse Source
This commit is contained in:
Andreas Rumpf
2026-06-15 21:20:09 +02:00
committed by GitHub
parent 7171e6f01f
commit acb9b3a4f1
29 changed files with 992 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
compiler/ast2nif.nim
View File

@@ -176,6 +176,7 @@ type
#writtenTypes: seq[PType] # types written in this module, to be unloaded later
#writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later
writtenPackages: HashSet[string]
depSuffixes: HashSet[string] # module suffixes already emitted as `(import ...)` deps
proc isLocalSym(sym: PSym): bool {.inline.} =
## Every symbol is emitted as a *global* (module-suffixed) name so that its
@@ -376,9 +377,21 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) =
# module scope and a template's open/mixin symbol of the same name resolves to
# the field instead of a local, producing "type mismatch: got 'T'". Fields are
# still indexed (for `obj.field` resolution via the loaded object type); they
# are merely not advertised as importable. `skEnumField` stays importable —
# enum values are legitimately usable as bare identifiers.
if sym.kindImpl != skField and {sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}:
# are merely not advertised as importable. Plain `skEnumField` stays importable
# — enum values are legitimately usable as bare identifiers — but a field of a
# `{.pure.}` enum is NOT: the source path keeps pure fields out of the importer
# scope (`declarePureEnumField`), reachable only qualified or via the restricted
# pure-enum mechanism (`importPureEnumFields`, fed by `ifaces[].pureEnums` which
# a loaded module rebuilds from its `PureEnumEntry` log ops). Marking them
# bare-importable made a loaded pure enum's fields leak into module scope
# (`populateInterfaceTablesFromIndex` adds every `x`/Exported sym to `interf`),
# e.g. nim-json-serialization's pure `JsonValueKind.Number` shadowing web3's
# `Number = distinct uint64` so `uint64(x).Number` failed under `nim ic`
# ("undeclared field 'Number'").
let isPureEnumField = sym.kindImpl == skEnumField and sym.typImpl != nil and
sym.typImpl.symImpl != nil and sfPure in sym.typImpl.symImpl.flagsImpl
if sym.kindImpl != skField and not isPureEnumField and
{sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}:
dest.addIdent "x"
else:
dest.addDotToken
@@ -546,6 +559,7 @@ proc trImport(w: var Writer; n: PNode) =
let fp = moduleSuffix(w.infos.config, s.positionImpl.FileIndex)
w.deps.addStrLit fp # raw string literal, no wrapper needed
w.deps.addParRi
w.depSuffixes.incl fp
proc trExport(w: var Writer; n: PNode) =
# Collect export information for the index
@@ -576,11 +590,13 @@ var repTraceTag = registerTag("reptrace")
var repDeepCopyTag = registerTag("repdeepcopy")
var repEnumToStrTag = registerTag("repenumtostr")
var repMethodTag = registerTag("repmethod")
var repPureEnumTag = registerTag("reppureenum")
#var repClassTag = registerTag("repclass")
var includeTag = registerTag("include")
var importTag = registerTag("import")
var implTag = registerTag("implementation")
var reexpModTag = registerTag("reexpmod")
var offerTag = registerTag("offer")
proc registerNifAstTags*() =
## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag`
@@ -606,10 +622,12 @@ proc registerNifAstTags*() =
repDeepCopyTag = registerTag("repdeepcopy")
repEnumToStrTag = registerTag("repenumtostr")
repMethodTag = registerTag("repmethod")
repPureEnumTag = registerTag("reppureenum")
includeTag = registerTag("include")
importTag = registerTag("import")
implTag = registerTag("implementation")
reexpModTag = registerTag("reexpmod")
offerTag = registerTag("offer")
proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) =
if n == nil:
@@ -726,8 +744,23 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) =
writeNode(w, dest, ast[i], forAst)
dec w.inProc
of nkImportStmt:
# this has been transformed for us, see `importer.nim` to contain a list of module syms:
trImport w, n
if w.inProc > 0:
# An `import` inside a template/macro/proc body — e.g. stew/importops'
# `tryImport`: `when compiles((; import v)): import v`. It is part of the
# body AST and must be serialized as a real node so the template
# re-expands it at each use site; it is NOT a module-level dependency
# edge (the import resolves where the template expands, against that
# module's deps). Diverting it to `w.deps` (the top-level path below)
# dropped it entirely: its child is the unexpanded template parameter
# `v`, not a module sym, so `trImport` wrote nothing and the body
# round-tripped EMPTY — a NIF-loaded `tryImport` then imported nothing.
w.withNode dest, n:
for i in 0 ..< n.len:
writeNode(w, dest, n[i], forAst)
else:
# top-level import: recorded as a dependency edge — `importer.nim` has
# already transformed `n` to contain a list of module syms.
trImport w, n
of nkIncludeStmt:
trInclude w, n
of nkExportStmt, nkExportExceptStmt:
@@ -821,6 +854,11 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) =
content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo)
content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo)
content.addParRi()
of PureEnumEntry:
content.addParLe repPureEnumTag, NoLineInfo
content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo)
content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo)
content.addParRi()
of GenericInstEntry:
discard "will only be written later to ensure it is materialized"
@@ -1202,7 +1240,11 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
opsLog: seq[LogEntry];
replayActions: seq[PNode] = @[];
implDeps: seq[int] = @[];
reexportedModules: seq[(string, string)] = @[]) =
reexportedModules: seq[(string, string)] = @[];
genericOffers: seq[tuple[generic, inst: PSym;
concreteTypes: seq[PType];
genericParamsCount: int]] = @[];
resolvedImportDeps: seq[FileIndex] = @[]) =
var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule)
var content = createTokenBuf(300)
@@ -1223,6 +1265,25 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
var bottom = createTokenBuf(300)
w.writeToplevelNode content, bottom, n
# Resolved import edges that left no syntactic `import` node in the top-level
# AST: an import generated INSIDE a `when` condition (e.g. stew/importops'
# `when tryImport x:` -> `when compiles((; import x)): import x`) really
# imports `x` — `addImportFileDep` recorded the edge in `graph.importDeps` —
# but the import node is folded away with the condition, so `trImport` never
# saw it and the NIF `deps` section omitted it. The backend closure walk
# (nifbackend.loadBackendModules) follows NIF `deps`, so without this edge a
# template-imported module's `{.compile.}`/`{.passL.}` directives never replay
# and its C/asm objects go unlinked (undefined `hashtree_hash`/`my_c_add` at
# link). Emit any resolved edge not already written as a syntactic import.
for f in resolvedImportDeps:
let fp = moduleSuffix(config, f)
if not w.depSuffixes.containsOrIncl(fp):
w.deps.addParLe importTag, NoLineInfo
w.deps.addDotToken # flags
w.deps.addDotToken # type
w.deps.addStrLit fp
w.deps.addParRi
# Re-exported MODULES (`import x; export x`): semExport puts only x's
# member syms into the nkExportStmt; the module sym itself reaches the
# exporter's interface via `reexportSym` and acts as a QUALIFIER there
@@ -1234,6 +1295,24 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
w.deps.addStrLit msuffix
w.deps.addParRi
# Generic-instance OFFERS: every generic instance this module created
# (`getOrDefault[MultiCodec]`, …). A consumer that re-instantiates the same
# generic must REUSE this instance instead of re-running `instantiateBody` in
# its own module scope — which lacks symbols visible only at the generic's
# definition site (e.g. a distinct type's `==` from the type's module), so
# operator/mixin resolution would fail ("type mismatch" at `hashcommon.rawGet`).
# The loader (modulegraphs.moduleFromNifFile) rebuilds `procInstCache` from
# these so `genericCacheGet` hits and the wrong-scope re-instantiation is
# skipped. Layout: (offer <genericSym> <instSym> <genericParamsCount> <type>...).
for off in genericOffers:
w.deps.addParLe offerTag, NoLineInfo
w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.generic)), NoLineInfo
w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(off.inst)), NoLineInfo
w.deps.addIntLit off.genericParamsCount
for ct in off.concreteTypes:
w.deps.addSymUse pool.syms.getOrIncl(typeToNifSym(ct, w.infos.config)), NoLineInfo
w.deps.addParRi
# the implTag is used to tell the loader that the
# bottom of the file is the implementation of the module:
content.addParLe implTag, NoLineInfo
@@ -1448,6 +1527,33 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string;
proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: string;
localSyms: var Table[string, PSym])
proc tryCreateTypeStub(c: var DecodeContext; t: SymId): PType =
## Like `createTypeStub` but returns nil instead of raising when the type has
## no offset in its module index (used by the best-effort `(offer …)` loader).
let name = pool.syms[t]
if not name.startsWith("`t"): return nil
result = c.types.getOrDefault(name)[0]
if result == nil:
var i = len("`t")
var k = 0
while i < name.len and name[i] in {'0'..'9'}:
k = k * 10 + name[i].ord - ord('0')
inc i
if i < name.len and name[i] == '.': inc i
var itemVal = 0'i32
while i < name.len and name[i] in {'0'..'9'}:
itemVal = itemVal * 10'i32 + int32(name[i].ord - ord('0'))
inc i
if i < name.len and name[i] == '.': inc i
let suffix = name.substr(i)
let id = itemId(moduleId(c, suffix).int32, itemVal)
let ii = addr c.mods[id.module.FileIndex].index
let offs = ii[].getOrDefault(name)
if offs.offset == 0:
return nil
result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial)
c.types[name] = (result, offs)
proc createTypeStub(c: var DecodeContext; t: SymId): PType =
let name = pool.syms[t]
assert name.startsWith("`t")
@@ -2128,6 +2234,11 @@ type
module*: PSym # set by modulegraphs.nim!
reexportedModules*: seq[(string, string)] # (name, suffix) of re-exported MODULE syms;
# materialized by modulegraphs.nim
genericOffers*: seq[tuple[generic, inst: PSym; concreteTypes: seq[PType];
genericParamsCount: int]]
## generic instances this module created; modulegraphs.nim rebuilds
## `procInstCache` from them so a consumer reuses the instance instead of
## re-instantiating it in its own (operator-blind) module scope.
proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) =
tok = next(s) # skip `(import`
@@ -2172,6 +2283,12 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag];
var t = next(s) # skip dot
var cont = true
let exportTag = pool.tags.getOrIncl"export"
# Top-level `let`/`var` sections are loaded even without LoadFullAst: they may
# declare `{.compileTime.}` globals whose VM slots the importer initializes
# eagerly (pipelines.initLoadedCompileTimeGlobals), which needs them visible in
# `topLevel`. They sit in the module header before `(implementation)`.
let letTag = pool.tags.getOrIncl(toNifTag(nkLetSection))
let varTag = pool.tags.getOrIncl(toNifTag(nkVarSection))
while cont and t.kind != EofToken:
if t.kind == ParLe:
if t.tagId == replayTag:
@@ -2210,6 +2327,8 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag];
t = loadLogOp(c, result.logOps, s, EnumToStrEntry, attachedTrace, module)
elif t.tagId == repMethodTag:
t = loadLogOp(c, result.logOps, s, MethodEntry, attachedTrace, module)
elif t.tagId == repPureEnumTag:
t = loadLogOp(c, result.logOps, s, PureEnumEntry, attachedTrace, module)
#elif t.tagId == repClassTag:
# t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module)
elif t.tagId == exportTag:
@@ -2269,10 +2388,39 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag];
t = next(s)
if mname.len > 0 and msuffix.len > 0:
result.reexportedModules.add (mname, msuffix)
elif t.tagId == offerTag:
# (offer <genericSym> <instSym> <genericParamsCount> <type>...) — see the
# writer. Resolve to PSyms/PTypes here; modulegraphs registers them into
# `procInstCache`. Best-effort: a type that fails to resolve drops the
# whole offer (the consumer then re-instantiates, the prior behaviour).
t = next(s) # skip (offer
var genSym, instSym: PSym = nil
var paramsCount = 0
var cts: seq[PType] = @[]
var idx = 0
var ok = true
while t.kind != ParRi and t.kind != EofToken:
if t.kind == Symbol:
if idx == 0: genSym = resolveHookSym(c, t.symId)
elif idx == 1: instSym = resolveHookSym(c, t.symId)
else:
let ct = tryCreateTypeStub(c, t.symId)
if ct == nil: ok = false
else: cts.add ct
inc idx
elif t.kind == IntLit:
paramsCount = int(pool.integers[t.intId])
t = next(s)
if t.kind != ParRi:
raiseAssert "expected ParRi in offer entry of module " & suffix
t = next(s)
if ok and genSym != nil and instSym != nil:
result.genericOffers.add (genSym, instSym, cts, paramsCount)
elif t.tagId == implTag:
cont = false
elif LoadFullAst in flags:
# Parse the full statement
elif LoadFullAst in flags or t.tagId == letTag or t.tagId == varTag:
# Parse the full statement. let/var sections are loaded unconditionally
# (see above) so `{.compileTime.}` globals reach the eager initializer.
var buf = createTokenBuf(50)
nextSubtree(s, buf, t)
t = next(s) # skip ParRi

3
compiler/astdef.nim
View File

@@ -986,7 +986,8 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode =
type
LogEntryKind* = enum
HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry
HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry,
PureEnumEntry
LogEntry* = object
kind*: LogEntryKind
op*: TTypeAttachedOp

6
compiler/commands.nim
View File

@@ -508,6 +508,7 @@ proc parseCommand*(command: string): Command =
of "jsonscript": cmdJsonscript
of "nifc": cmdNifC # generate C from NIF files
of "ic": cmdIc # generate .build.nif for nifmake
of "icconfig": cmdIcConfig # produce the precompiled config artifact
else: cmdUnknown
proc setCmd*(conf: ConfigRef, cmd: Command) =
@@ -960,6 +961,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
# config loading can replay it instead of re-parsing the `nim.cfg` chain.
expectArg(conf, switch, arg, pass, info)
conf.icPreparsedConfig = arg
of "icconfigout":
# `nim icconfig` only: where to write the precompiled config artifact (see
# options.icConfigOut). The `nim ic` driver spawns the producer with this.
expectArg(conf, switch, arg, pass, info)
conf.icConfigOut = arg
of "icbackendstage":
# `nim nifc` only: per-module backend stage, one of cg|merge|emit (see
# options.icBackendStage). Empty (switch unused) keeps the whole-program

378
compiler/deps.nim
View File

@@ -11,8 +11,8 @@
## This enables incremental and parallel compilation using the `m` switch.
import std / [os, tables, sets, times, osproc, algorithm, strtabs, strutils, syncio]
import options, msgs, lineinfos, pathutils, condsyms, icconfig,
modulepaths, extccomp, cnif
import options, msgs, lineinfos, pathutils, condsyms,
modulepaths, extccomp, cnif, platform
import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder]
import "../dist/nimony/src/gear2" / modnames
@@ -196,8 +196,11 @@ proc resolveInclude(c: DepContext; origin, toResolve: string): string =
proc traverseDeps(c: var DepContext; pair: FilePair; current: Node)
proc processInclude(c: var DepContext; includePath: string; current: Node) =
let resolved = resolveInclude(c, current.files[current.files.len - 1].nimFile, includePath)
proc processInclude(c: var DepContext; includePath: string; current: Node; origin: string) =
# `origin` = the file the `include` literally appears in (an included file's
# own nested includes/imports must resolve relative to IT, not the importing
# module's main file).
let resolved = resolveInclude(c, origin, includePath)
if resolved.len == 0 or not fileExists(resolved):
return
@@ -211,8 +214,25 @@ proc processInclude(c: var DepContext; includePath: string; current: Node) =
traverseDeps(c, c.toPair(resolved), current)
discard c.includeStack.pop()
proc processImport(c: var DepContext; importPath: string; current: Node) =
let resolved = resolveImport(c, current.files[0].nimFile, importPath)
proc getsImplicitImports(c: DepContext; nimFile: string): bool =
## Mirror the compiler's `belongsToStdlib` guard (pipelines.nim): `--import:X`
## (conf.implicitImports) is applied only to NON-stdlib modules. The scanner
## must agree, otherwise it edges a stdlib module → X that the compiler never
## actually creates, fabricating a cycle that folds X — and the modules X
## claims to produce — into the system SCC (whose `nim m` is driven from
## system.nim and never reaches them). Stdlib == under conf.libpath.
not isRelativeTo(nimFile, c.config.libpath.string)
proc processImport(c: var DepContext; importPath: string; current: Node; origin: string) =
# `origin` = the file the `import` literally appears in. Crucial for imports
# inside `include`d files: e.g. `system.nim` includes `system/excpt.nim`, which
# does `import stacktraces` — that must resolve relative to `excpt.nim`
# (lib/system/) → `lib/system/stacktraces.nim`, NOT relative to `system.nim`
# (lib/) which has no `stacktraces.nim`. Resolving against the main file silently
# dropped the `system → stacktraces` edge, so stacktraces was a separate SCC in
# the static round and got re-grouped (and recompiled with divergent type ids)
# only after the post-sem `.s.deps` revealed the edge.
let resolved = resolveImport(c, origin, importPath)
if resolved.len == 0 or not fileExists(resolved):
return
@@ -226,12 +246,15 @@ proc processImport(c: var DepContext; importPath: string; current: Node) =
# Every module depends on system.nim
if c.systemNodeId >= 0:
newNode.deps.add c.systemNodeId
# ... and on every `--import`ed module (conf.implicitImports). A `--import`ed
# module is itself imported by its own closure (which also gets these edges),
# so the cycle folds into one strongly-connected component (see computeSCCs),
# just like system.nim's closure.
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
# ... and on every `--import`ed module (conf.implicitImports), but only for
# the non-stdlib modules the compiler actually applies implicit imports to
# (see getsImplicitImports). A `--import`ed module is imported by its own
# non-stdlib closure (which also gets these edges), so that cycle folds into
# one small strongly-connected component (see computeSCCs) instead of being
# smeared across system + stdlib.
if getsImplicitImports(c, pair.nimFile):
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
c.processedModules[pair.modname] = newNode.id
c.nodes.add newNode
traverseDeps(c, pair, newNode)
@@ -251,10 +274,44 @@ proc skipSubtree(s: var Stream; first: PackedToken) =
elif t.kind == ParRi: dec depth
elif t.kind == EofToken: return
proc evalCondIdent(c: DepContext; v: string): bool =
## Truth value of a bare identifier appearing in a `when` condition.
type
CondVal = enum
## Tri-state truth of a `when` condition as the static scanner sees it.
## `cvUnknown` is the crucial state: the scanner can't determine the value
## (an arbitrary call like `compiles`/`tryImport`, an unknown const ident,
## an unresolvable comparison). A dependency scanner must NEVER drop a real
## import, so callers treat `cvUnknown` as "keep the dependency". The bug
## this replaces: everything-unknown collapsed to `true`, and `not true`
## is `false`, so an `else:` branch (emitted as `when (not COND)`) silently
## dropped its imports (e.g. `when tryImport x: ... else: import x`, or
## system's `else: include excpt` hiding `import stacktraces`).
cvFalse, cvTrue, cvUnknown
proc toCondVal(b: bool): CondVal = (if b: cvTrue else: cvFalse)
proc condNot(a: CondVal): CondVal =
case a
of cvFalse: cvTrue
of cvTrue: cvFalse
of cvUnknown: cvUnknown
proc condAnd(a, b: CondVal): CondVal =
if a == cvFalse or b == cvFalse: cvFalse
elif a == cvTrue and b == cvTrue: cvTrue
else: cvUnknown
proc condOr(a, b: CondVal): CondVal =
if a == cvTrue or b == cvTrue: cvTrue
elif a == cvFalse and b == cvFalse: cvFalse
else: cvUnknown
proc evalCondIdent(c: DepContext; v: string): CondVal =
## Truth value of a bare identifier appearing in a `when` condition. Unknown
## idents are `cvUnknown` (kept), not `true` — so `when not SOMEIDENT:` no
## longer drops its import.
case v
of "false": false
of "true": cvTrue
of "false": cvFalse
of "hasThreadSupport":
# system.nim's `hasThreadSupport` is `compileOption("threads") and
# not defined(nimscript)`; the conservative `true` would schedule the
@@ -262,12 +319,12 @@ proc evalCondIdent(c: DepContext; v: string): bool =
# whose NIFs a --threads:off compile never produces — nifmake then
# sees missing outputs and re-runs the system rule (and everything
# downstream) on every rerun.
optThreads in c.config.globalOptions
toCondVal(optThreads in c.config.globalOptions)
of "usesDestructors":
# system.nim's `usesDestructors = defined(gcDestructors) or
# defined(gcHooks)`; guards mmdisp.nim's `include "system/gc"` whose
# transitive imports (sharedlist, locks) an orc compile never produces.
isDefined(c.config, "gcDestructors") or isDefined(c.config, "gcHooks")
toCondVal(isDefined(c.config, "gcDestructors") or isDefined(c.config, "gcHooks"))
of "isMainModule":
# Only the project main module is compiled with `isMainModule` true; an
# imported module's `when isMainModule` blocks are dead. The conservative
@@ -275,171 +332,123 @@ proc evalCondIdent(c: DepContext; v: string): bool =
# `tools/grammar_nanny`, a node that gets a cg rule but is never linked,
# so the merge stage can pick it as a shared def's owner -> undefined
# symbols at link).
c.scanningMain
else: true
toCondVal(c.scanningMain)
else: cvUnknown
proc evalCondExpr(c: DepContext; s: var Stream): bool =
## Read exactly one condition expression from `s` and return its truth
## value. Consumes tokens whether the expression is recognised or not so
## the caller stays in sync. Recognises `defined(IDENT)`, the boolean
## operators `not`/`and`/`or`, and the literals `true`/`false`. Anything
## else (e.g. a call to an arbitrary proc) is treated as `true` — the
## conservative direction, since a false negative here drops a real
## dependency from the build graph.
proc constIdentValue(c: DepContext; ident: string): string =
## String value of a compile-time platform constant that appears in `when`
## guards, or "" when unknown. Mirrors the compiler's magics so the scanner
## evaluates e.g. `when hostOS == "standalone"` the SAME way the real compile
## does. Without this the comparison is "unknown" → the conservative `true`,
## which is WRONG once negated (`else:` branches emit `not (==)`), so a real
## conditional `include`/`import` is dropped (e.g. system's `else: include
## excpt`, hiding `import stacktraces`).
# Must match the compiler's magics EXACTLY, incl. case: `hostOS`/`hostCPU` etc.
# fold to the lower-cased platform name (see semfold.nim mHostOS/mHostCPU), and
# user code compares against lower-case literals (`when hostOS == "linux"`).
case ident
of "hostOS": result = toLowerAscii(platform.OS[c.config.target.targetOS].name)
of "hostCPU": result = toLowerAscii(platform.CPU[c.config.target.targetCPU].name)
of "buildOS": result = toLowerAscii(platform.OS[c.config.target.hostOS].name)
of "buildCPU": result = toLowerAscii(platform.CPU[c.config.target.hostCPU].name)
else: result = ""
proc readOperandValue(c: DepContext; s: var Stream): string =
## Read one operand of an `==`/`!=` infix and return its string value (a string
## literal verbatim, a platform-constant ident resolved, anything else ""), fully
## consuming the operand (subtrees are skipped) so the caller stays in sync.
let t = next(s)
case t.kind
of StringLit: result = pool.strings[t.litId]
of Ident: result = constIdentValue(c, pool.strings[t.litId])
of ParLe:
result = ""
skipSubtree(s, t)
else: result = ""
proc evalCondCmp(c: DepContext; s: var Stream; isEq: bool): CondVal =
## Evaluate `a == b` / `a != b`. Both operands known → real result; otherwise
## `cvUnknown` (so a negated comparison keeps, not drops, the dependency).
let v1 = readOperandValue(c, s)
let v2 = readOperandValue(c, s)
if v1.len > 0 and v2.len > 0:
result = toCondVal((v1 == v2) == isEq)
else:
result = cvUnknown
proc evalCondExpr(c: DepContext; s: var Stream; t: PackedToken): CondVal
proc readCond(c: DepContext; s: var Stream): CondVal =
## Read one full condition subtree (its own opener included) and evaluate it.
let t = next(s)
evalCondExpr(c, s, t)
proc evalCondExpr(c: DepContext; s: var Stream; t: PackedToken): CondVal =
## Evaluate the condition whose opening token `t` has ALREADY been read,
## consuming the rest of the expression so the caller stays in sync.
## Recognises `defined(IDENT)`, `not`/`and`/`or`, `==`/`!=` and the literals
## `true`/`false`; everything else (an arbitrary call such as `compiles` /
## `tryImport`, an unknown const) is `cvUnknown`. Both negation-sensitive
## (`not cvUnknown == cvUnknown`) and short-circuit-free: `and`/`or` always
## read both operands so the stream stays in sync regardless of the result.
case t.kind
of Ident:
result = evalCondIdent(c, pool.strings[t.litId])
of ParLe:
let tag = pool.tags[t.tagId]
# For prefix/infix/call nodes the operator name is the first child; for a
# bare `(not ...)`/`(and ...)`/`(or ...)`/`(par ...)` node the tag itself is
# the operator and the operands follow directly.
var name = tag
case tag
of "call", "cmd", "callstrlit", "infix", "prefix":
# First child is the head (function/operator name).
let head = next(s)
var name = ""
if head.kind == Ident: name = pool.strings[head.litId]
case name
of "defined":
let arg = next(s)
var sym = ""
if arg.kind == Ident: sym = pool.strings[arg.litId]
result = sym.len > 0 and isDefined(c.config, sym)
of "not":
result = not evalCondExpr(c, s)
of "and":
result = evalCondExpr(c, s)
if result: result = evalCondExpr(c, s)
else: skipSubtree(s, next(s))
of "or":
result = evalCondExpr(c, s)
if not result: result = evalCondExpr(c, s)
else: skipSubtree(s, next(s))
else:
result = true
# Drain whatever remains until the matching ParRi.
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
else: name = ""
else: discard
case name
of "defined":
let arg = next(s)
var sym = ""
if arg.kind == Ident: sym = pool.strings[arg.litId]
result = toCondVal(sym.len > 0 and isDefined(c.config, sym))
of "not":
result = not evalCondExpr(c, s)
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
result = condNot(readCond(c, s))
of "and":
result = evalCondExpr(c, s)
if result: result = evalCondExpr(c, s)
else: skipSubtree(s, next(s))
# consume closing ParRi
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
let a = readCond(c, s)
let b = readCond(c, s)
result = condAnd(a, b)
of "or":
result = evalCondExpr(c, s)
if not result: result = evalCondExpr(c, s)
else: skipSubtree(s, next(s))
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
let a = readCond(c, s)
let b = readCond(c, s)
result = condOr(a, b)
of "==", "!=":
result = evalCondCmp(c, s, name == "==")
of "par":
# a parenthesised grouping such as `(defined(a) or defined(b))`: evaluate
# the inner expression. Without this, `par` fell through to the `else`
# branch below and evaluated to `true`, which silently inverted conditions
# like `not (defined(macosx) or defined(bsd))` and dropped real imports
# (e.g. `cpuinfo`'s conditional `import std/posix`).
result = evalCondExpr(c, s)
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
# a parenthesised grouping such as `(defined(a) or defined(b))`.
result = readCond(c, s)
else:
skipSubtree(s, t)
result = true
result = cvUnknown
# Drain whatever remains until the matching ParRi.
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
else:
result = true
result = cvUnknown
proc whenMarkerHolds(c: DepContext; s: var Stream): bool =
proc whenMarkerHolds(c: DepContext; s: var Stream): CondVal =
## Caller has just consumed the `(when` ParLe. Read children until the
## matching `)`, AND-ing each evaluated condition.
result = true
## matching `)`, AND-ing each evaluated condition. Returns the tri-state
## result; callers keep the dependency unless it is provably `cvFalse`.
result = cvTrue
while true:
# peek by reading; if it's ParRi, we're done
let t = next(s)
if t.kind == ParRi: return
if t.kind == EofToken: return
if t.kind == ParLe:
# Re-feed by manually evaluating the subtree starting at `t`.
# evalCondExpr expects to read its own opener, so handle it directly.
let tag = pool.tags[t.tagId]
case tag
of "call", "cmd", "callstrlit", "infix", "prefix":
let head = next(s)
var name = ""
if head.kind == Ident: name = pool.strings[head.litId]
var ok = true
case name
of "defined":
let arg = next(s)
var sym = ""
if arg.kind == Ident: sym = pool.strings[arg.litId]
ok = sym.len > 0 and isDefined(c.config, sym)
of "not":
ok = not evalCondExpr(c, s)
of "and":
ok = evalCondExpr(c, s)
if ok: ok = evalCondExpr(c, s)
of "or":
ok = evalCondExpr(c, s)
if not ok: ok = evalCondExpr(c, s)
else:
ok = true
# finish the subtree
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
if not ok: result = false
of "not", "and", "or":
# Re-emit a synthetic dispatch: rewrap by descending.
var ok = true
case tag
of "not":
ok = not evalCondExpr(c, s)
of "and":
ok = evalCondExpr(c, s)
if ok: ok = evalCondExpr(c, s)
of "or":
ok = evalCondExpr(c, s)
if not ok: ok = evalCondExpr(c, s)
else: discard
var depth = 1
while depth > 0:
let n = next(s)
if n.kind == ParLe: inc depth
elif n.kind == ParRi: dec depth
elif n.kind == EofToken: return
if not ok: result = false
else:
# Unknown — treat as true and skip.
skipSubtree(s, t)
elif t.kind == Ident:
if not evalCondIdent(c, pool.strings[t.litId]): result = false
# a true / unknown ident keeps the current result
result = condAnd(result, evalCondExpr(c, s, t))
proc parseImportPath(s: var Stream; t: var PackedToken): seq[string] =
## Parse an import path expression and return the list of module paths it
@@ -543,8 +552,12 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) =
var live = true
if t.kind == ParLe and pool.tags[t.tagId] == "when":
# whenMarkerHolds consumes everything up to and including the
# closing `)` of the `(when ...)` subtree.
live = whenMarkerHolds(c, s)
# closing `)` of the `(when ...)` subtree. Drop the import only when
# the condition is PROVABLY false; a `cvUnknown` condition (e.g. an
# `else:` branch guarded by `not <unevaluatable call>`, as in
# `when tryImport x: ... else: import x`) keeps the dependency so the
# static graph never misses a real import.
live = whenMarkerHolds(c, s) != cvFalse
t = next(s)
if not live:
# Drain the rest of this import/include node.
@@ -568,15 +581,15 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) =
# be treated as modules. Both still create a real dependency on `m`.
for importPath in parseImportPath(s, t):
if importPath.len > 0:
processImport(c, importPath, current)
processImport(c, importPath, current, pair.nimFile)
else:
while t.kind != ParRi and t.kind != EofToken:
for importPath in parseImportPath(s, t):
if importPath.len > 0:
if tag == "include":
processInclude(c, importPath, current)
processInclude(c, importPath, current, pair.nimFile)
else:
processImport(c, importPath, current)
processImport(c, importPath, current, pair.nimFile)
# Drain any remaining tokens of this node (e.g. the symbol list of a
# `fromimport`), up to and including the node's closing ')'.
var depth = 1
@@ -694,15 +707,14 @@ proc computeForwardedArgs(c: DepContext): seq[string] =
# then abort builds the whole-program compilation accepts. Forward the
# real project so children filter diagnostics identically.
result.add "--icproject:" & c.config.projectFull.string
# Precompiled config: serialise the driver's config once and have every
# child replay it instead of re-parsing the `nim.cfg` chain and re-running
# `config.nims` in the VM. See compiler/icconfig.nim. `-d:icNoPreparsedConfig`
# restores the old per-child config parsing (for bisecting a suspected
# config-replay divergence without clearing caches).
if not isDefined(c.config, "icNoPreparsedConfig"):
let cfgArtifact = nimcache / "ic_config.cfg.nif"
writeIcConfig(c.config, cfgArtifact)
result.add "--icPreparsedConfig:" & cfgArtifact
# Precompiled config: every child replays the one artifact produced (in a
# separate `nim icconfig` process) and already replayed by the driver itself —
# see `icconfig.ensureIcConfig`, run before the driver's own `loadConfigs`. So
# `nim ic` is always governed by this single artifact, for speed and so the
# driver and its children agree by construction. Forward the path the driver
# replayed (`conf.icPreparsedConfig`); `commandIc` has already guaranteed it
# exists, else it bailed.
result.add "--icPreparsedConfig:" & c.config.icPreparsedConfig
proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
## Frontend build file: the nifler (parse) and `nim m` (sem) rules only. The
@@ -1029,6 +1041,12 @@ proc commandIc*(conf: ConfigRef) =
rawMessage(conf, errGenerated, "nifler tool not found. Install nimony or add nifler to PATH.")
return
# Resolve the `.nim` source first, exactly like `wantMainModule`. Without
# this, an extensionless project arg (`nim ic path/to/foo`) resolves to a
# same-named sibling that already exists — e.g. the ELF a prior `nim c`
# left behind — and nifler chokes on the binary (`invalid token \127`,
# ELF magic). `addFileExt` only appends when there is no extension.
conf.projectFull = addFileExt(conf.projectFull, NimExt)
let projectFile = conf.projectFull.string
if not fileExists(projectFile):
rawMessage(conf, errGenerated, "project file not found: " & projectFile)
@@ -1118,6 +1136,15 @@ proc commandIc*(conf: ConfigRef) =
# from its importer — and rerun; nifmake's mtime pruning keeps completed
# work. A round that discovers nothing new but still fails is a real error.
let forwardedArgs = computeForwardedArgs(c)
# The precompiled config drives every `nim m`/`nim nifc` child and the driver
# itself (`ensureIcConfig` produced it and `loadConfigs` replayed it). If it
# is not on disk something went wrong producing it — children would each
# silently fall back to re-parsing the whole config chain — so refuse to
# continue without it.
if conf.icPreparsedConfig.len == 0 or not fileExists(conf.icPreparsedConfig):
rawMessage(conf, errGenerated,
"precompiled config missing: " & conf.icPreparsedConfig)
return
let nifmake = findNifmake()
# Build the per-module rules concurrently: nifmake fans out all commands at
# each DAG depth via execProcesses (defaults to all cores). Cold builds are
@@ -1164,8 +1191,9 @@ proc commandIc*(conf: ConfigRef) =
let newNode = Node(files: @[pair], id: c.nodes.len)
if c.systemNodeId >= 0:
newNode.deps.add c.systemNodeId
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
if getsImplicitImports(c, pair.nimFile):
for impId in c.implicitNodeIds:
if impId != newNode.id: newNode.deps.add impId
c.processedModules[pair.modname] = newNode.id
c.nodes.add newNode
idx = newNode.id

12
compiler/ic/replayer.nim
View File

@@ -19,8 +19,11 @@ import std/tables
when defined(nimPreviewSlimSystem):
import std/assertions
proc replayStateChanges*(module: PSym; g: ModuleGraph) =
let list = module.ast
proc replayStateChanges*(module: PSym; g: ModuleGraph; list: PNode) =
## `list` is an `nkStmtList` of `nkReplayAction` nodes (macro-cache puts/incs/
## adds/incls and a few pragmas) recorded for `module`. Under the NIF backend a
## loaded module's `ast` is never reconstructed, so the caller passes the replay
## actions it parsed out of the module's NIF directly.
assert list != nil
assert list.kind == nkStmtList
for n in list:
@@ -64,8 +67,9 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) =
g.cacheTables[destKey] = initBTree[string, PNode]()
if not contains(g.cacheTables[destKey], key):
g.cacheTables[destKey].add(key, val)
else:
internalError(g.config, n.info, "key already exists: " & key)
# else: the same key was already replayed. Under IC the import closure is
# replayed (direct module + transitive deps), so the same registration can
# legitimately be reached twice; re-applying it is a no-op, not an error.
of "incl":
let destKey = n[1].strVal
let val = n[2]

168
compiler/icconfig.nim
View File

@@ -31,23 +31,39 @@
## `--path` arguments; replaying their raw, config-dir-relative arguments here
## would misresolve.
import options, commands, lineinfos
import std/[algorithm, os, sets]
import options, commands, lineinfos, pathutils, msgs
import std/[algorithm, os, sets, osproc, times, streams, syncio]
import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse]
const
IcConfigVersion* = "1"
IcConfigVersion* = "2"
## Artifact format version. Bump on any layout change here so a child built
## by an older compiler rejects a stale artifact and falls back to normal
## config loading instead of replaying a format it cannot parse.
proc writeIcConfig*(conf: ConfigRef; outfile: string) =
## Serialise the config-file switches recorded during `loadConfigs` plus the
## resolved `cppDefines` set into the artifact at `outfile`.
var b = nifbuilder.open(outfile)
## Serialise the resolved config (the config-file switches recorded during
## `loadConfigs`, the resolved `cppDefines`/`searchPaths`, the nimcache dir, and
## the list of config *source* files for staleness detection) into `outfile`.
## `OnlyIfChanged`: when the content is byte-identical to what is already on
## disk the file is left untouched so its mtime does not advance — otherwise
## every `nim ic` run would re-fire the whole nifmake graph (see `nifler`'s
## `produceConfig`, whose model this mirrors).
var b = nifbuilder.open(outfile, writeMode = OnlyIfChanged)
b.withTree "stmts":
b.withTree "meta":
b.addStrLit IcConfigVersion
b.withTree "sources":
# Every config file read while loading (nim.cfg chain + config.nims), so a
# later run can decide via mtimes whether this artifact is still current
# (see `sourcesChanged`).
for f in conf.configFiles:
b.addStrLit f.string
b.withTree "nimcache":
# Resolved build nimcache. Recorded (unlike the path-search switches) so the
# driver, which replays this artifact instead of parsing `nim.cfg`, still
# learns a `--nimcache:` set inside `nim.cfg` and builds in the right place.
b.addStrLit conf.nimcacheDir.string
b.withTree "cppdefines":
# HashSet iteration order is unspecified; sort so the artifact is
# byte-stable across runs (nifmake keys rebuilds off content changes).
@@ -55,6 +71,15 @@ proc writeIcConfig*(conf: ConfigRef; outfile: string) =
for d in conf.cppDefines: defs.add d
sort defs
for d in defs: b.addStrLit d
b.withTree "searchpaths":
# The resolved (absolute) search paths. Path-search *switches* are skipped
# below because their raw arguments are config-dir-relative; the net effect
# lives here instead, so a replayer with no `--path` command-line arguments
# (the `nim ic` driver itself) still resolves imports. `nim m`/`nim nifc`
# children also receive these as forwarded `--path` args; the dedup on
# replay makes the overlap harmless.
for p in conf.searchPaths:
b.addStrLit p.string
b.withTree "switches":
for sw in conf.icConfigSwitches:
b.addTree "sw"
@@ -74,7 +99,10 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
let
stmtsTag = tags.registerTag("stmts")
metaTag = tags.registerTag("meta")
sourcesTag = tags.registerTag("sources")
nimcacheTag = tags.registerTag("nimcache")
cppTag = tags.registerTag("cppdefines")
pathsTag = tags.registerTag("searchpaths")
switchesTag = tags.registerTag("switches")
swTag = tags.registerTag("sw")
var buf = parseFromFile(infile, 1000, pool, tags)
@@ -95,6 +123,22 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
inc c
else:
skip c
elif c.cursorTagId == nimcacheTag:
c.loopInto:
if c.kind == StrLit:
let nc = strVal(c)
# Only when nimcache was not already pinned on the command line: a
# `--nimcache:` argument the driver/child was launched with must win
# over whatever `nim.cfg` recorded into the artifact.
if nc.len > 0 and conf.nimcacheDir.isEmpty:
conf.nimcacheDir = AbsoluteDir(nc)
inc c
else:
skip c
elif c.cursorTagId == sourcesTag:
# Replay does not need the source list; it exists only for
# `sourcesChanged`. Skip the whole section.
skip c
elif c.cursorTagId == cppTag:
c.loopInto:
if c.kind == StrLit:
@@ -102,6 +146,17 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
inc c
else:
skip c
elif c.cursorTagId == pathsTag:
c.loopInto:
if c.kind == StrLit:
# Append preserving the serialised order (which already reflects the
# driver's addPath insert-at-front sequence), deduping against any
# path a child already received via a forwarded `--path` argument.
let d = AbsoluteDir(strVal(c))
if not conf.searchPaths.contains(d): conf.searchPaths.add d
inc c
else:
skip c
elif c.cursorTagId == switchesTag:
c.loopInto:
if c.kind == TagLit and c.cursorTagId == swTag:
@@ -125,3 +180,104 @@ proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
skip c
endRead(c)
result = sawMeta and version == IcConfigVersion
proc sourcesChanged*(configFile: string): bool =
## True when the precompiled config at `configFile` is missing, malformed,
## written by an incompatible version, or any recorded config *source* file is
## newer than it (or has vanished) — i.e. the artifact must be regenerated.
## Mirrors nifler's `sourcesChanged`: the source list lives inside the artifact
## so this needs no out-of-band knowledge of which `nim.cfg`s were read.
if not fileExists(configFile): return true
let modtime = getLastModificationTime(configFile)
var pool = newPool()
var tags = newTagPool()
let
stmtsTag = tags.registerTag("stmts")
metaTag = tags.registerTag("meta")
sourcesTag = tags.registerTag("sources")
var buf = parseFromFile(configFile, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return true
var version = ""
var depsChanged = false
c.loopInto:
if c.kind == TagLit and c.cursorTagId == metaTag:
c.loopInto:
if c.kind == StrLit:
version = strVal(c)
inc c
else:
skip c
elif c.kind == TagLit and c.cursorTagId == sourcesTag:
c.loopInto:
if c.kind == StrLit:
let dep = strVal(c)
if not fileExists(dep) or getLastModificationTime(dep) >= modtime:
depsChanged = true
inc c
else:
skip c
else:
skip c
endRead(c)
result = depsChanged or version != IcConfigVersion
proc produceIcConfig*(conf: ConfigRef) =
## The `cmdIcConfig` command. By the time it runs, the normal pipeline has
## already fully parsed the `nim.cfg` chain and run `config.nims`, so the
## resolved config is sitting in `conf`; just serialise it to `--o`.
let outPath = conf.icConfigOut
if outPath.len == 0:
rawMessage(conf, errGenerated, "icconfig: missing output path (--icConfigOut)")
return
createDir(parentDir(outPath))
writeIcConfig(conf, outPath)
proc ensureIcConfig*(conf: ConfigRef) =
## Driver-side (`cmdIc`). Make sure an up-to-date precompiled config exists,
## (re)producing it in a *separate* process when missing or stale, then point
## `conf.icPreparsedConfig` at it so the driver replays the very same config its
## `nim m`/`nim nifc` children will — perfect speed (config parsed at most once,
## skipped entirely when nothing changed) and consistency (one producer, every
## process replays its output). The artifact lives in the nimcache derived from
## the command line (pre-config-parse), which is the one the children are told;
## a `--nimcache:` set inside `nim.cfg` is recovered from the artifact itself.
let cacheDir = getNimcacheDir(conf).string
# Start from a clean cache when the on-disk NIF format stamp is absent or stale
# (see `icFormatVersion`). This must happen HERE, before the config artifact is
# produced — `commandIc` performs the same check later, but by then the artifact
# would already live in the cache and the wipe would delete it.
createDir(cacheDir)
let versionFile = cacheDir / "ic.version"
let stamp = if fileExists(versionFile): readFile(versionFile) else: ""
if stamp != icFormatVersion:
removeDir(cacheDir)
createDir(cacheDir)
writeFile(versionFile, icFormatVersion)
let outPath = cacheDir / "ic_config.cfg.nif"
if not fileExists(outPath) or sourcesChanged(outPath):
createDir(cacheDir)
# Re-invoke ourselves as the config producer: reuse this process's command
# line, dropping the command argument (`ic`) in favour of `icconfig` and the
# explicit output path, both BEFORE the project file (anything after the
# project is swallowed into `config.arguments` by `cmdLineRest`). The
# producer re-reads `nim.cfg` itself.
var pargs = @["icconfig", "--icConfigOut:" & outPath]
var droppedCmd = false
for a in commandLineParams():
if not droppedCmd and a.len > 0 and a[0] != '-':
droppedCmd = true # drop the original command token (`ic`)
else:
pargs.add a
let p = startProcess(getAppFilename(), args = pargs,
options = {poStdErrToStdOut})
let outp = p.outputStream.readAll()
let code = p.waitForExit()
p.close()
if code != 0 or not fileExists(outPath):
rawMessage(conf, errGenerated,
"failed to produce precompiled config (exit code " & $code & "):\n" & outp)
return
conf.icPreparsedConfig = outPath

6
compiler/main.nim
View File

@@ -29,6 +29,7 @@ when defined(nimPreviewSlimSystem):
import ../dist/checksums/src/checksums/sha1
import pipelines
from icconfig import produceIcConfig
when not defined(nimKochBootstrap):
import nifbackend
@@ -439,6 +440,11 @@ proc mainCommand*(graph: ModuleGraph) =
commandIc(conf)
else:
rawMessage(conf, errGenerated, "nim deps not available in bootstrap build")
of cmdIcConfig:
# Produce the precompiled config artifact for `nim ic` (config already
# parsed by the normal pipeline); a separate process spawned by the driver.
wantMainModule(conf)
produceIcConfig(conf)
of cmdParse:
wantMainModule(conf)
discard parseFile(conf.projectMainIdx, cache, conf)

38
compiler/modulegraphs.nim
View File

@@ -146,6 +146,11 @@ type
cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API; IC: implemented
cacheCounters*: Table[string, BiggestInt] # IC: implemented
cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
transitiveReplayActions*: seq[PNode] # macro-cache replay actions collected from
# the transitive import closure of a NIF-loaded module (loadTransitiveHooks);
# the caller (pipelines) replays them so a dependency's macrocache state — e.g.
# nim-serialization's flavor registration — reaches a module that imports it
# only indirectly. Drained per moduleFromNifFile call.
passes*: seq[TPass]
pipelinePass*: PipelinePass
onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
@@ -944,6 +949,14 @@ when not defined(nimKochBootstrap):
if not g.hookClosure.containsOrIncl(fileIdx.int):
let precomp = loadNifModule(ast.program, suffix, interf, interfHidden, {})
registerLoadedHooks(g, precomp.logOps)
# Collect the dependency's macro-cache replay actions (put/inc/add/incl)
# so the importer being compiled also sees macrocache state registered
# by a transitively-imported module. Pragma replay actions are a backend
# concern and are intentionally not collected here.
for n in precomp.topLevel:
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
g.transitiveReplayActions.add n
for d in precomp.deps: stack.add d
proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym =
@@ -1009,6 +1022,15 @@ when not defined(nimKochBootstrap):
if ms != nil:
strTableAdd(g.ifaces[fileIdx.int].interf, ms)
# Rebuild `procInstCache` from this module's generic-instance OFFERS so a
# consumer's `genericCacheGet` finds the instance and SKIPS re-running
# `instantiateBody` in its own module scope (which lacks symbols visible only
# at the generic's definition site — see ast2nif's `(offer …)`).
for off in result.genericOffers:
g.procInstCache.mgetOrPut(off.generic.itemId, @[]).add PInstantiation(
sym: off.inst, concreteTypes: off.concreteTypes,
genericParamsCount: off.genericParamsCount, compilesId: 0)
# Mark module as cached
g.cachedMods.incl fileIdx.int
g.hookClosure.incl fileIdx.int
@@ -1019,6 +1041,11 @@ when not defined(nimKochBootstrap):
case x.kind
of ConverterEntry:
g.ifaces[fileIdx.int].converters.add x.sym
of PureEnumEntry:
# rebuild the pure-enum list (source path: `addPureEnum`) so importers can
# offer this loaded `{.pure.}` enum's fields as the restricted pure-enum
# fallback (`importPureEnumFields`).
g.ifaces[fileIdx.int].pureEnums.add x.sym
of MethodEntry:
discard "dispatch buckets already rebuilt by registerLoadedHooks"
of GenericInstEntry:
@@ -1063,7 +1090,16 @@ proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
proc belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool =
## Check if symbol belongs to the 'stdlib' package.
sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId
# Compare the package *name* (an interned ident), not the package symbol's
# `.id`. Under per-module IC (`nim m`) the system module is loaded from a NIF
# in a process that does not compile it from source, so its package symbol is
# reconstructed with a fresh `.id` that no longer matches the freshly-interned
# package of a stdlib module compiled standalone here — making the old id
# comparison wrongly report `false` and inject `--import`ed modules into the
# stdlib. Both are canonically named `stdlib` (lib/stdlib.nimble); in a normal
# `nim c` build (system compiled from source) the ids match too, so this is a
# no-op there.
sym.getPackageSymbol.name.id == graph.systemModule.getPackageSymbol.name.id
proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): SuggestFileSymbolDatabase =
result = graph.suggestSymbols.getOrDefault(fileIdx, newSuggestFileSymbolDatabase(fileIdx, optIdeExceptionInlayHints in graph.config.globalOptions))

9
compiler/nim.nim
View File

@@ -29,6 +29,7 @@ import
pathutils, modulegraphs
from ast2nif import registerNifAstTags
from icconfig import ensureIcConfig
from std/browsers import openDefaultBrowser
from nodejs import findNodeJs
@@ -114,6 +115,14 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
self.processCmdLineAndProjectPath(conf)
# `nim ic` driver: ensure the precompiled config exists (produced by a separate
# `nim icconfig` process, skipped when nothing changed) BEFORE config loading,
# so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the
# driver runs on the exact same config its children will. See icconfig.nim.
when not defined(nimKochBootstrap):
if conf.cmd == cmdIc:
ensureIcConfig(conf)
var graph = newModuleGraph(cache, conf)
if not self.loadConfigsAndProcessCmdLine(cache, conf, graph):
return

12
compiler/nimconf.nim
View File

@@ -246,10 +246,14 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile
proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) =
setDefaultLibpath(conf)
# `nim ic` children replay the precompiled config the driver recorded once,
# instead of re-reading the `nim.cfg` chain and re-running `config.nims` in the
# VM. A missing/format-incompatible artifact returns false: fall through to
# normal config loading so an older child or a deleted cache still works.
# The `nim ic` driver and its `nim m`/`nim nifc` children replay the precompiled
# config (produced once by a separate `nim icconfig` process — see
# `icconfig.ensureIcConfig`, which sets `icPreparsedConfig` for the driver
# before this runs; the children get it as a forwarded `--icPreparsedConfig`
# argument) instead of re-reading the `nim.cfg` chain and re-running
# `config.nims` in the VM. A missing/format-incompatible artifact returns false:
# fall through to a normal parse (this is also the path the `nim icconfig`
# producer itself takes, since it runs with no `icPreparsedConfig`).
if conf.icPreparsedConfig.len > 0 and applyIcConfig(conf, conf.icPreparsedConfig):
return
template readConfigFile(path) =

11
compiler/options.nim
View File

@@ -200,6 +200,7 @@ type
cmdCompileToNif
cmdNifC # generate C code from NIF files
cmdIc # generate .build.nif for nifmake
cmdIcConfig # `nim ic`'s precompiled-config producer (writes ic_config.cfg.nif)
const
cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,
@@ -418,12 +419,16 @@ type
# module's package the "main package" and unfilter
# foreign-package diagnostics; the real project
# restores whole-program filtering semantics.
icPreparsedConfig*: string # under `nim m`/`nim nifc`: path of the precompiled
# config artifact written once by the `nim ic` driver.
icPreparsedConfig*: string # under the `nim ic` driver and its `nim m`/`nim nifc`
# children: path of the precompiled config artifact.
# When set, `loadConfigs` replays the recorded
# config-file switches from it instead of re-reading
# the `nim.cfg` chain and re-running `config.nims`
# (which the VM makes expensive) per subprocess.
# (which the VM makes expensive) per process. The
# artifact itself is produced by a separate
# `nim icconfig` process (see `cmdIcConfig`).
icConfigOut*: string # under `nim icconfig`: the path to write the
# precompiled config artifact to (set via `--o`).
icConfigSwitches*: seq[tuple[switch, arg: string]]
# the config-file (`passPP`) switches applied while
# loading config, in order. Recorded by every nim

105
compiler/pipelines.nim
View File

@@ -269,23 +269,94 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
# the union; intra-group entries are filtered by the writer.
var implDeps: seq[int] = @[]
for id in graph.icImplDeps: implDeps.add id
# Generic-instance OFFERS: every instance THIS module created, so a
# consumer reuses it rather than re-instantiating in its own scope (which
# cannot see symbols visible only at the generic's definition site — e.g.
# a distinct type's `==`). See ast2nif.writeNifModule / moduleFromNifFile.
var genericOffers: seq[tuple[generic, inst: PSym;
concreteTypes: seq[PType]; genericParamsCount: int]] = @[]
for genItemId, instList in graph.procInstCache:
for inst in instList:
if inst.sym != nil and inst.sym.itemId.module == module.position and
inst.sym.instantiatedFrom != nil and inst.compilesId == 0:
# `concreteTypes` is pre-sized to `paramsLen+gp.len`; a tail slot can
# stay nil (e.g. fewer materialized params than `paramsLen`). Such an
# offer can't be serialized — skip it (the consumer re-instantiates,
# the prior behaviour) rather than emit a nil type reference.
var hasNil = false
for ct in inst.concreteTypes:
if ct == nil: hasNil = true; break
if not hasNil:
genericOffers.add (inst.sym.instantiatedFrom, inst.sym,
inst.concreteTypes, inst.genericParamsCount)
# The module's REAL resolved direct imports (incl. macro/template-generated
# ones with no surviving syntactic node). Passed to writeNifModule so the
# NIF `deps` section is complete (the backend closure walk needs it), and
# reused below for the `.s.deps` sidecar (frontend graph re-derivation).
let resolvedImportDeps = graph.importDeps.getOrDefault(module.position.FileIndex, @[])
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps, reexportedModuleSyms(graph, module))
replayActions, implDeps, reexportedModuleSyms(graph, module),
genericOffers, resolvedImportDeps)
# The module's REAL direct imports (incl. macro-generated) for `nim ic`'s
# graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep.
var semDepPaths: seq[string] = @[]
for f in graph.importDeps.getOrDefault(module.position.FileIndex, @[]):
for f in resolvedImportDeps:
semDepPaths.add toFullPath(graph.config, f)
writeSemDeps(graph.config, module.position.int32, semDepPaths)
result = true
proc loadedDefSym(defs: PNode): PSym =
## The defined symbol of a let/var entry as it loads back from a NIF: the
## section child is a bare `nkSym` (the `(sd …)` reference), but be defensive
## about the from-source shapes too (`nkIdentDefs`, a pragma-wrapped name).
case defs.kind
of nkSym: result = defs.sym
of nkPragmaExpr:
result = if defs.len > 0: loadedDefSym(defs[0]) else: nil
of nkIdentDefs, nkConstDef:
result = if defs.len > 0: loadedDefSym(defs[0]) else: nil
else: result = nil
proc initLoadedCompileTimeGlobals(graph: ModuleGraph; module: PSym; topLevel: PNode) =
## Eagerly initialize the compile-time globals (`let/var {.compileTime.}`) of a
## module restored from a NIF. In a normal sem these VM slots are filled by
## `setupCompileTimeVar` (semstmts) as the section is semchecked; a NIF-loaded
## module is never semchecked, so without this a macro or compile-time proc that
## reads such a global finds a nil slot. The lazy `vmgen.genGlobalInit` fallback
## is order-fragile across proc boundaries (it emits the init at the first
## VM-gen'd reference, which need not be the first one executed), so the init has
## to happen here, once, before any of the module's code can run. The symbol's
## own `ast` is the `nkIdentDefs` (initializer included); re-wrap it in a section
## exactly as semstmts does and hand it to the same evaluator.
if topLevel == nil: return
let idgen = idGeneratorFromModule(module)
for stmt in topLevel:
if stmt.kind notin {nkLetSection, nkVarSection}: continue
for defs in stmt:
let s = loadedDefSym(defs)
if s != nil and s.kind in {skLet, skVar} and
{sfCompileTime, sfGlobal} <= s.flags and
s.ast != nil and s.ast.kind == nkIdentDefs:
var sect = newNodeI(stmt.kind, s.info)
sect.add s.ast
setupCompileTimeVar(module, idgen, graph, sect)
proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags; fromModule: PSym = nil): PSym =
var flags = flags
if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
result = graph.getModule(fileIdx)
template processModuleAux(moduleStatus) =
when defined(icDbg):
block:
let dbgf = open("/tmp/defdbg.txt", fmAppend)
dbgf.writeLine toFullPath(graph.config, fileIdx) &
" nimStackTraceOverride=" & $isDefined(graph.config, "nimStackTraceOverride") &
" nimscript=" & $isDefined(graph.config, "nimscript") &
" optCompress=" & $(optCompress in graph.config.globalOptions) &
" cmd=" & $graph.config.cmd
dbgf.close()
onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
var s: PLLStream = nil
if sfMainModule in flags:
@@ -333,9 +404,33 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
if sfSystemModule in flags:
graph.systemModule = result
partialInitModule(result, graph, fileIdx, AbsoluteFile(toFullPath(graph.config, fileIdx)))
# Replay state changes from the loaded NIF module
if result.ast != nil:
replayStateChanges(result, graph)
# Replay the module's recorded state changes: macro-cache operations
# (std/macrocache puts/incs/adds/incls) plus a few pragmas. The loader
# parsed them into `precomp.topLevel` (mixed with other top-level nodes),
# so filter to the replay actions. A loaded module's `ast` is never
# rebuilt, so this used to be skipped (`result.ast == nil`) and a
# NIF-loaded module's macro cache was lost — e.g. nim-serialization's
# flavor registration became invisible to dependents (`DefaultFlavor:
# automatic serialization is not enabled`).
var replayList = newNodeI(nkStmtList, result.info)
for n in precomp.topLevel:
# Only macro-cache ops (put/inc/add/incl). The pragma replay actions
# (compile/link/passc/hint/...) are a backend/link concern handled by
# the nifc closure, and re-emitting a loaded module's hints/warnings on
# every import would be wrong — so they are deliberately skipped here.
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
replayList.add n
# Plus the macro-cache actions of the module's transitive import closure
# (collected by the moduleFromNifFile call above via loadTransitiveHooks),
# so a flavor/type registered in an indirectly-imported module is visible.
for n in graph.transitiveReplayActions: replayList.add n
graph.transitiveReplayActions.setLen 0
if replayList.len > 0:
replayStateChanges(result, graph, replayList)
# Fill the VM slots of the module's `{.compileTime.}` globals now (sem
# would have, but a NIF-loaded module is never semchecked).
initLoadedCompileTimeGlobals(graph, result, precomp.topLevel)
return result # Return early, don't process from source
let path = toFullPath(graph.config, fileIdx)
let filename = AbsoluteFile path

7
compiler/semcall.nim
View File

@@ -983,7 +983,12 @@ proc explicitGenericSym(c: PContext, n: PNode, s: PSym, errors: var CandidateErr
diagnostics: m.diagnostics))
return nil
var newInst = generateInstance(c, s, m.bindings, n.info)
newInst.typ.excl tfUnresolved
# `generateInstance` may return an instance REUSED from another module's NIF
# `(offer …)` — its type is Sealed (immutable). Such an instance is already
# fully resolved (`tfUnresolved` cleared at its original instantiation), so the
# `excl` is a no-op; skip it rather than assert on a Sealed-type mutation.
if newInst.typ.state != Sealed:
newInst.typ.excl tfUnresolved
let info = getCallLineInfo(n)
markUsed(c, info, s, isGenericInstance = false)
onUse(info, s, isGenericInstance = false)

7
compiler/semdata.nim
View File

@@ -410,6 +410,13 @@ proc addConverterDef*(c: PContext, conv: PSym) =
proc addPureEnum*(c: PContext, e: PSym) =
assert e != nil
add(c.graph.ifaces[c.module.position].pureEnums, e)
# record for IC: a NIF-loaded module rebuilds `Iface.pureEnums` from these log
# entries (moduleFromNifFile); without it a loaded module's pure enums were
# invisible to importers, so `importPureEnumFields` never offered their fields
# and unqualified pure-enum values stopped resolving. (Same pattern as
# `addConverterDef`.)
c.graph.opsLog.add LogEntry(kind: PureEnumEntry, module: c.module.position,
key: "", sym: e)
proc addPattern*(c: PContext, p: PSym) =
assert p != nil

20
compiler/semexprs.nim
View File

@@ -1905,6 +1905,22 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode =
n.typ = n.typ.elementType
result.add(n)
proc markResultVarIsPtr(c: PContext, x: PNode) {.inline.} =
## Set `tfVarIsPtr` on the (result) sym node's type. Under IC that type can be a
## NIF-loaded (Sealed) and interned instance which must not be mutated in place
## (it could corrupt other users of the shared type, and the assert forbids it):
## give this result its own copy carrying the flag, exactly like a from-source
## compile has a fresh result type here.
if tfVarIsPtr in x.typ.flags: return
if x.typ.state == Sealed:
let fresh = copyType(x.typ, c.idgen, x.typ.owner)
fresh.incl tfVarIsPtr
x.typ = fresh
if x.kind == nkSym and x.sym.state != Sealed:
x.sym.typ = fresh
else:
x.typ.incl tfVarIsPtr
proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
if le.kind == nkHiddenDeref:
var x = le[0]
@@ -1912,10 +1928,10 @@ proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
if x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView):
n[0] = x # 'result[]' --> 'result'
n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent)
x.typ.incl tfVarIsPtr
markResultVarIsPtr(c, x)
#echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info
elif sfGlobal in x.sym.flags:
x.typ.incl tfVarIsPtr
markResultVarIsPtr(c, x)
proc borrowCheck(c: PContext, n, le, ri: PNode) =
const

11
compiler/sighashes.nim
View File

@@ -149,6 +149,17 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi
# as properties are accessed and trigger lazy loading.
backendEnsureMutable(t)
# Bare type-class keywords used as a typedesc without arguments (e.g. `array`,
# `range`, `distinct` passed to `signatureHash`) have no children, so the
# structural branches below would index a non-existent `elementType`. Hash them
# by kind (+ sym for an extra, stable distinction) — enough for a stable,
# distinct identity. (`seq`/`openArray`/`tuple` already fall through the empty
# `else` loop unharmed; this covers the branches that index `elementType`.)
if t.kind in {tyArray, tyRange, tyDistinct} and not t.hasElementType:
c &= char(t.kind)
if t.sym != nil: c.hashSym(t.sym)
return
case t.kind
of tyGenericInvocation:
for a in t.kids:

16
compiler/vm.nim
View File

@@ -1955,7 +1955,21 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
if regs[rb].node.kind != nkSym:
stackTrace(c, tos, pc, "node is not a symbol")
else:
regs[ra].node.strVal = $sigHash(regs[rb].node.sym, c.config)
let shSym = regs[rb].node.sym
# When `signatureHash` is applied to a type (e.g. a `T: typedesc`/generic
# param), hash the *type* it denotes, not the parameter symbol. Hashing the
# symbol routes through `hashNonProc`, which mixes in `s.disamb` — a
# per-module instantiation counter. Under incremental compilation the
# registering module and a consuming module instantiate the surrounding
# generic separately, get different `disamb`s, and produce different
# hashes for the same type (nim-serialization's auto-serialization lookup
# missed because of this). Hashing the underlying type via `hashType` is
# type-identity based and stable across the NIF boundary.
let shTyp = shSym.typ
if shTyp != nil and shTyp.kind == tyTypeDesc and shTyp.hasElementType:
regs[ra].node.strVal = $hashType(shTyp.elementType, c.config)
else:
regs[ra].node.strVal = $sigHash(shSym, c.config)
of opcSlurp:
decodeB(rkNode)
createStr regs[ra]

9
compiler/vmgen.nim
View File

@@ -1782,8 +1782,15 @@ proc genGlobalInit(c: PCtx; n: PNode; s: PSym) =
# This is rather hard to support, due to the laziness of the VM code
# generator. See tests/compile/tmacro2 for why this is necessary:
# var decls{.compileTime.}: seq[NimNode] = @[]
# Load the slot's ADDRESS (not its value): the lazy initializer must REPLACE
# the null slot, which `opcWrDeref` only does for an `rkNodeAddr` target
# (`nAddr[] = n` for refs). With `opcLdGlobal` the slot value is loaded and for
# a ref-typed global that value is an `nkNilLit` ("nil ref"); writing through it
# hits the VM's nil-deref guard ("attempt to access a nil address"). This path
# is reached for compile-time globals whose defining module is restored from a
# NIF under `nim ic` (so `setupCompileTimeVar` never ran to eagerly init them).
let dest = c.getTemp(s.typ)
c.gABx(n, opcLdGlobal, dest, s.position)
c.gABx(n, opcLdGlobalAddr, dest, s.position)
if s.astdef != nil:
let tmp = c.genx(s.astdef)
c.genAdditionalCopy(n, opcWrDeref, dest, 0, tmp)

29
koch.nim
View File

@@ -600,19 +600,38 @@ proc xtemp(cmd: string) =
finally:
copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe)
proc icTest(args: string) =
temp("")
let inp = os.parseCmdLine(args)[0]
proc runIcTestFile(inp: string) =
## Compile a single `tests/ic` file with `nim ic`, once per `#!EDIT!#` fragment
## (each fragment is the file's source after that incremental edit). Only checks
## that `nim ic` exits 0 — the produced binary's output is not verified here.
let content = readFile(inp)
let nimExe = getAppDir() / "bin" / "nim_temp".exe
var i = 0
for fragment in content.split("#!EDIT!#"):
let file = inp.replace(".nim", "_temp.nim")
writeFile(file, fragment)
var cmd = nimExe & " ic --hint:Conf:off --warnings:off "
cmd.add quoteShell(file)
exec(cmd)
inc i
# The `tests/ic` files that `nim ic` must keep compiling. Multi-module tests rely
# on a sibling helper (`timp` -> `myimp`, `tcompiletimeglobal` -> `mctglobal`),
# which exercises the NIF import/load path the single-file tests do not.
const icSuite = ["thallo", "tconverter", "timp", "tmiscs", "tparseutils",
"tcompiletimeglobal", "tsighashstable", "tpureenum", "tgenericoffer"]
proc icTest(args: string) =
temp("")
let parsed = os.parseCmdLine(args)
if parsed.len > 0 and parsed[0].len > 0:
# `koch ic <file>`: run just that file.
runIcTestFile(parsed[0])
else:
# `koch ic`: the full regression set we want to keep working — the test
# suite plus both self-host bootstraps (`bootic` and `bootic -d:release`).
for t in icSuite:
runIcTestFile("tests" / "ic" / (t & ".nim"))
bootic("", skipIntegrityCheck = false)
bootic("-d:release", skipIntegrityCheck = false)
proc buildDrNim(args: string) =
if not dirExists("dist/nimz3"):

28
tests/ic/mctglobal.nim Normal file
View File

@@ -0,0 +1,28 @@
# Helper module for tcompiletimeglobal.nim (not a test itself; no `discard`).
#
# Exercises `{.compileTime.}` module-level globals across the NIF boundary: under
# `nim ic` this module is compiled to its own NIF and *loaded* (not semchecked)
# by the importer, so its compile-time globals must be eagerly initialized at
# load time. A macro that splices such a global into a `quote do:` otherwise
# reads a nil VM slot.
import macros
let injectedName {.compileTime.} = ident "ctgValue"
proc genAssign*(): NimNode =
## The order-fragile case: the CT global is read inside a proc that the macro
## calls, not in the macro's own `quote`. The lazy VM init attaches to the
## first vmgen'd reference, which need not be the first one executed.
result = quote do:
`injectedName` = 42
macro defineCtgValue*(): untyped =
result = newStmtList()
# Read the global via the helper proc FIRST, then via the macro's own quote,
# so the proc's reference executes before the macro's: only eager init at load
# time makes both see the initialized value.
let assign = genAssign()
result.add quote do:
var `injectedName`: int
result.add assign

3
tests/ic/mgofmc.nim Normal file
View File

@@ -0,0 +1,3 @@
# Helper for tgenericoffer.nim: a distinct type whose `==` lives in THIS module.
type MultiCodec* = distinct int
proc `==`*(a, b: MultiCodec): bool = int(a) == int(b)

10
tests/ic/mgofmh.nim Normal file
View File

@@ -0,0 +1,10 @@
# Helper for tgenericoffer.nim: imports mgofmc, builds a compile-time const
# Table[MultiCodec,int] and exposes a GENERIC proc whose (T-independent) body
# instantiates getOrDefault[MultiCodec] here, where mgofmc's `==` is visible.
import tables, mgofmc
proc buildTab(): Table[MultiCodec, int] =
result[MultiCodec(1)] = 100
result[MultiCodec(2)] = 200
const CodeHashes = buildTab()
proc digest*[T](x: T): int =
CodeHashes.getOrDefault(MultiCodec(2))

6
tests/ic/mgofpid.nim Normal file
View File

@@ -0,0 +1,6 @@
# Helper for tgenericoffer.nim: imports mgofmh but NOT mgofmc, then calls the
# generic `digest`. Under IC this re-instantiates digest here, where mgofmc's
# `==` is NOT in scope — so getOrDefault[MultiCodec] must be REUSED from mgofmh's
# offer, not re-instantiated, or `==(MultiCodec)` resolution fails.
import mgofmh
proc callDigest*(): int = digest(5)

21
tests/ic/mpureenum.nim Normal file
View File

@@ -0,0 +1,21 @@
# Helper module for tpureenum.nim (not a test itself; no `discard`).
#
# Defines a `{.pure.}` enum whose field name (`Number`) collides with a distinct
# type of the same name. Under `nim ic` the pure enum is loaded from a NIF; its
# fields must NOT leak into the importer's unqualified scope (the source path
# keeps them out via `declarePureEnumField`). Before the fix, a loaded pure
# enum's fields were marked bare-importable and leaked into `interf`, so the
# field shadowed the distinct type and `uint64(x).Number` failed with
# "undeclared field 'Number' for type system.uint64" (the nim-json-serialization
# `JsonValueKind.Number` vs web3 `Number = distinct uint64` bug).
type
Number* = distinct uint64
JsonValueKind* {.pure.} = enum
String, Number, Object, Array, Bool, Null
proc val*(x: Number): uint64 = uint64(x)
proc toNumber*(x: uint64): Number =
x.Number # must bind to the distinct TYPE `Number`, not the pure enum field

48
tests/ic/msighashstable.nim Normal file
View File

@@ -0,0 +1,48 @@
# Helper module for tsighashstable.nim (not a test itself; no `discard`).
#
# Models nim-serialization's auto-serialization registry: a flavor records which
# types it auto-serializes in a `std/macrocache` keyed by `signatureHash(T)`,
# computed through a generic `{.compileTime.}` func. The registration happens
# here (at this module's compile time); the lookup happens in the importer.
#
# Under `nim ic` the two modules are compiled separately, so the generic
# `getSig[T]` is instantiated independently on each side. `signatureHash` must
# therefore hash the *type* `T` denotes, not the generic parameter symbol — the
# latter mixes in a per-module `disamb` counter and diverges across the NIF
# boundary, making the lookup miss.
import std/[macrocache, macros, typetraits]
type DefaultFlavor* = object
macro calcSig*(T: typed): untyped =
doAssert(T.typeKind == ntyTypeDesc)
result = newLit(signatureHash(T))
func getSig*(F: type DefaultFlavor, T: distinct type): string {.compileTime.} =
calcSig(T)
func getTable*(F: type DefaultFlavor): CacheTable {.compileTime.} =
CacheTable("nsrzStableTable" & typetraits.name(F))
func setAuto*(F: type DefaultFlavor, T: distinct type) {.compileTime.} =
let sig = F.getSig(T)
let table = F.getTable()
if not table.hasKey(sig):
table[sig] = newLit(1)
func getAuto*(F: type DefaultFlavor, T: distinct type): bool {.compileTime.} =
let sig = F.getSig(T)
let table = F.getTable()
table.hasKey(sig)
template autoCheck*(F: distinct type, T: distinct type, body) =
when not F.getAuto(T):
{.error: "auto serialization not enabled for `" & typetraits.name(T) & "`".}
else:
body
static:
setAuto(DefaultFlavor, string)
setAuto(DefaultFlavor, SomeInteger)
setAuto(DefaultFlavor, seq)

16
tests/ic/tcompiletimeglobal.nim Normal file
View File

@@ -0,0 +1,16 @@
discard """
output: '''42'''
"""
# Regression test: `{.compileTime.}` module-level globals of a NIF-loaded module
# must be initialized so macros/compile-time procs that splice them produce valid
# code. Before the eager-init fix this failed under `nim ic` with
# "attempt to access a nil address" / "illformed AST: break nil" /
# "undeclared identifier". `koch ic` only checks the compile succeeds; a nil
# global makes `defineCtgValue` emit `var <nil>: int` / `<nil> = 42` and the
# compile fails.
import mctglobal
defineCtgValue()
echo ctgValue

12
tests/ic/tgenericoffer.nim Normal file
View File

@@ -0,0 +1,12 @@
discard """
output: '''200'''
"""
# Regression test for the `(offer …)` generic-instance sharing across the NIF
# boundary. A generic loaded from a NIF whose body uses a type-bound op on a
# CONCRETE type (here getOrDefault on Table[MultiCodec,int]) must be reused, not
# re-instantiated in the consumer's scope which lacks the type's `==`. Before the
# fix this failed under `nim ic` with `hashcommon.nim type mismatch ... MultiCodec`.
import mgofpid
echo callDigest()

21
tests/ic/tpureenum.nim Normal file
View File

@@ -0,0 +1,21 @@
discard """
output: '''5
Number
Object'''
"""
# Regression test: a `{.pure.}` enum loaded from a NIF must keep its fields out
# of the importer's unqualified scope (no leak), while still being reachable
# qualified AND via the restricted unambiguous-bare pure-enum fallback
# (`importPureEnumFields`, fed by `ifaces[].pureEnums` which the loader rebuilds
# from `PureEnumEntry` log ops).
#
# Before the fix the pure fields leaked into `interf` under `nim ic`, so
# `JsonValueKind.Number` shadowed the distinct `Number` type and the conversion
# below failed: "undeclared field 'Number' for type system.uint64".
import mpureenum
echo toNumber(5'u64).val # distinct conversion: field must NOT shadow type
echo JsonValueKind.Number # qualified pure-enum access still works
echo Object # unambiguous bare pure-enum field still resolves

33
tests/ic/tsighashstable.nim Normal file
View File

@@ -0,0 +1,33 @@
discard """
output: '''ok string
ok int
ok seq'''
"""
# Regression test: `signatureHash(T)` must be stable across the NIF boundary so
# that a macrocache keyed by it (nim-serialization's auto-serialization registry)
# can be populated in one module and queried from another under `nim ic`.
#
# Before the fix, `signatureHash` hashed the generic *parameter symbol* (whose
# `disamb` is a per-module instantiation counter) instead of the type it denotes.
# The registering module (msighashstable) and this importer instantiated the
# generic `getSig[T]` separately, got different `disamb`s, and the lookups for
# `string`/`SomeInteger` missed -> `{.error: auto serialization not enabled.}`.
import msighashstable
proc writeStr(F: type DefaultFlavor, v: string) =
autoCheck(F, string):
echo "ok string"
proc writeInt[T: SomeInteger](F: type DefaultFlavor, v: T) =
autoCheck(F, SomeInteger):
echo "ok int"
proc writeSeq[T](F: type DefaultFlavor, v: seq[T]) =
autoCheck(F, seq):
echo "ok seq"
writeStr(DefaultFlavor, "hi")
writeInt(DefaultFlavor, 42)
writeSeq(DefaultFlavor, @[1, 2, 3])