diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 66d1c2be95..9ccf6d9a7e 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -12,6 +12,9 @@ import std / [assertions, tables, sets] from std / strutils import startsWith, endsWith, contains from std / os import fileExists +from std / syncio import readFile +from std / algorithm import sort +import "../dist/checksums/src/checksums" / sha1 import astdef, idents, msgs, options import lineinfos as astli import pathutils #, modulegraphs @@ -807,9 +810,378 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = of GenericInstEntry: discard "will only be written later to ensure it is materialized" +# --------------------------- Interface cookie --------------------------- +# +# Port of Nimony's `processForChecksum` (dist/nimony/src/lib/nifindexes.nim): +# ONE checksum per module over the importer-visible surface, stored in a tiny +# `.iface.nif` sidecar written OnlyIfChanged. deps.nim points the +# dependents' `nim_m` build edges at the sidecar instead of the bulky semmed +# NIF, so nifmake's mtime pruning stops the m-step cascade at the first +# module whose interface did not change. +# +# Hashed (importer-visible surface): +# - import/include/export entries, `(replay ...)` macro-cache actions and the +# rep* hook/converter/enumtostr registrations (all eagerly consumed by every +# importer's sem via processTopLevel/loadTransitiveHooks). +# - every EXPORTED `(sd ...)`: full content for consts/types/vars/lets and for +# routines with inline semantics — templates, macros, iterators, generics +# (explicit `(genericparams)` in the routine ast or, for implicitly generic +# procs, `tyGenericParam`-kinded types in the signature) and `inline`-callconv +# procs. Plain procs/funcs/methods/converters hash the signature only, the +# body is skipped. (Nimony hashes generic bodies only via the `.inline` path; +# we close that gap here — generic bodies are instantiated by importers.) +# - nothing else: private defs and top-level init code are invisible to +# importers' sem (their effects on dependents' CODEGEN are covered by the +# nifc backend's transitive NIF-mtime invalidation, which is unchanged). +# +# Token-content hashing only — line infos never enter the hash. Names DEFINED +# inside a hashed (sd) (params, locals, the embedded `(td `tK.item.mod)` defs) +# are replaced by per-sd ordinals and module-local `tK.item references are +# replaced by their structural td hash: both carry process-local mint counters +# that shift file-wide when an unrelated body creates a new type (measured: +# a single new instantiation renumbered every later signature), while +# dependents never reference them by name (verified over a full compiler +# cache: cross-module refs hit only top-level routine names). +# +# The cookie finally mixes in the DIRECT dependencies' sidecar contents +# ("hash chaining"): an interface change then propagates transitively +# level-by-level even when an intermediate module's own surface is unchanged +# (its sem still consumed the dep's surface, e.g. via the transitive hook +# replay). Chaining also guarantees a fired rule refreshes its sidecar mtime, +# which nifmake's max-output `needsRebuild` needs to not re-fire forever. +# +# The IMPL cookie (`.impl.nif`) complements it: a line-info-free hash +# of the module's ENTIRE content with the iface cookie mixed in. Dependents +# that consumed this module's bodies at compile time (recorded in the +# `.edges.nif` sidecar; see `ModuleGraph.icImplDeps`) are gated on it instead. + +type + CookieCtx = object + selfSuffix: string + tdRanges: Table[string, int] # td name -> start of its first (td ...) tree + memo: Table[string, string] # td name -> structural digest + expanding: HashSet[string] # cycle guard for recursive td expansion + depSuffixes: seq[string] # module suffixes of the direct imports + +proc nextTree(buf: TokenBuf; i: int): int = + ## Index just past the atom or balanced subtree starting at `i`. + result = i+1 + if buf[i].kind != ParLe: return + var nested = 0 + var j = i + while j < buf.len: + case buf[j].kind + of ParLe: inc nested + of ParRi: + dec nested + if nested == 0: return j+1 + else: discard + inc j + result = buf.len + +proc updateAtom(s: var Sha1State; t: PackedToken) = + # mirrors nimony's nifchecksums.update: token content only, no line infos + case t.kind + of ParLe: + s.update "(" + s.update pool.tags[t.tagId] + of ParRi: s.update ")" + of Ident: + s.update " " + s.update pool.strings[t.litId] + of StringLit: + s.update " \"" + s.update pool.strings[t.litId] + of IntLit: + s.update " " + s.update $pool.integers[t.intId] + of UIntLit: + s.update " " + s.update $pool.uintegers[t.uintId] + of FloatLit: + # hash the bit pattern, not a formatted float (no formatting variance) + s.update " f" + s.update $cast[uint64](pool.floats[t.floatId]) + of CharLit: + s.update " c" + s.update $t.uoperand + of DotToken: s.update "." + of UnknownToken: s.update "?" + of EofToken: s.update "!" + of Symbol, SymbolDef: discard "handled by hashRegion" + +proc isModuleLocalName(c: CookieCtx; name: string): bool = + let sn = parseSymName(name) + result = sn.module.len == 0 or sn.module == c.selfSuffix + +proc hashRegion(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; + start, theEnd: int; skipFrom = -1; skipTo = -1; + keepFirstDefLiteral = false) + +proc expandTd(c: var CookieCtx; buf: TokenBuf; name: string): string = + ## Structural digest of a module-local type def: hashes the `(td ...)` tree + ## instead of the volatile `tK.item counter name. Memoized; cycles fall back + ## to the literal name (sound — at worst a spurious cookie change). + if c.memo.hasKey(name): return c.memo[name] + if not c.tdRanges.hasKey(name) or c.expanding.contains(name): + return name + c.expanding.incl name + let start = c.tdRanges[name] + var sub = newSha1State() + hashRegion(sub, c, buf, start, nextTree(buf, start)) + result = "&" & $SecureHash(sub.finalize()) + c.expanding.excl name + c.memo[name] = result + +proc hashRegion(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; + start, theEnd: int; skipFrom = -1; skipTo = -1; + keepFirstDefLiteral = false) = + # pass 1: assign ordinals to every symbol DEFINED in the hashed region + # (params, locals, embedded type defs). The region's own top-level name + # (first SymbolDef) stays literal when requested — it is what importers + # reference. + var ords = initTable[string, int]() + var first = keepFirstDefLiteral + var i = start + while i < theEnd: + if i == skipFrom: + i = skipTo + continue + if buf[i].kind == SymbolDef: + let name = pool.syms[buf[i].symId] + if first: + first = false + elif isModuleLocalName(c, name) and not ords.hasKey(name): + ords[name] = ords.len + inc i + # pass 2: hash + first = keepFirstDefLiteral + i = start + while i < theEnd: + if i == skipFrom: + i = skipTo + continue + let t = buf[i] + if t.kind in {Symbol, SymbolDef}: + let name = pool.syms[t.symId] + s.update(if t.kind == SymbolDef: " :" else: " ") + if t.kind == SymbolDef and first: + first = false + s.update name + elif ords.hasKey(name): + s.update "%" + s.update $ords[name] + elif name.startsWith("`t") and isModuleLocalName(c, name): + s.update expandTd(c, buf, name) + else: + s.update name + else: + updateAtom s, t + inc i + +proc scanSigTypeMarkers(buf: TokenBuf; start, theEnd: int): bool = + ## True if the routine's serialized signature marks it as inline-semantics: + ## an `inline` calling convention or a `tyGenericParam`-kinded type + ## (implicitly generic proc). + let gpPrefix = "`t" & $ord(tyGenericParam) & "." + var i = start + while i < theEnd: + let t = buf[i] + if t.kind == Ident and pool.strings[t.litId] == "inline": + return true + if t.kind in {Symbol, SymbolDef} and pool.syms[t.symId].startsWith(gpPrefix): + return true + inc i + result = false + +proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): int = + ## Contributes one `(sd ...)` subtree to the cookie; returns the index past it. + result = nextTree(buf, start) + if buf[start+1].kind != SymbolDef: return + let marker = buf[start+2] + if not (marker.kind == Ident and pool.strings[marker.litId] == "x"): + return # not importable -> invisible to dependents' sem (nimony parity) + # field layout, see writeSymDef: kind magic flags options offset position + # annex type owner ast loc constraint instantiatedFrom + var fields: array[13, int] = default(array[13, int]) + var i = start + 3 + for f in 0 ..< 13: + fields[f] = i + i = nextTree(buf, i) + var kind = skUnknown + {.cast(uncheckedAssign).}: + kind = parse(TSymKind, pool.tags[buf[fields[0]].tagId]) + var skipFrom = -1 + var skipTo = -1 + if kind in {skProc, skFunc, skMethod, skConverter}: + var fullBody = scanSigTypeMarkers(buf, fields[7], fields[8]) + let ast = fields[9] + if not fullBody and buf[ast].kind == ParLe: + # routine ast tree: tag flags type name pattern genericParams params ... + var p = ast + 1 # flags atom + p = nextTree(buf, p) # -> type slot + p = nextTree(buf, p) # -> son 0 (name) + p = nextTree(buf, p) # -> son 1 (pattern) + p = nextTree(buf, p) # -> son 2 (genericParams) + if buf[p].kind == ParLe and + pool.tags[buf[p].tagId] == toNifTag(nkGenericParams): + fullBody = true + if not fullBody and buf[ast].kind == ParLe: + # skip son `bodyPos` (6) of the routine ast tree; NOT the last element — + # sem appends the result sym at `resultPos` (7) after the body. + let astEnd = nextTree(buf, ast) + var p = ast + 1 # the flags atom + var ok = true + for _ in 0 ..< 2 + bodyPos: # flags, type, sons 0..5 + p = nextTree(buf, p) + if p >= astEnd - 1: + ok = false + break + if ok: + skipFrom = p + skipTo = nextTree(buf, p) + # templates/macros/iterators and all non-routine kinds (consts carry their + # value, types their structure incl. default field values): hash everything. + hashRegion(s, c, buf, start, result, skipFrom, skipTo, keepFirstDefLiteral = true) + +proc scanStmtsForCookie(s: var Sha1State; c: var CookieCtx; buf: TokenBuf) = + ## Walks the whole written module, hashing only the importer-visible pieces; + ## unknown structure is descended into (var/let/type section wrappers, + ## top-level code) but contributes nothing itself — nimony-style. + let exportTag = pool.tags.getOrIncl(toNifTag(nkExportStmt)) + let exportExceptTag = pool.tags.getOrIncl(toNifTag(nkExportExceptStmt)) + var i = 0 + while i < buf.len: + let t = buf[i] + if t.kind == ParLe: + let tid = t.tagId + if tid == sdefTag: + i = cookieSd(s, c, buf, i) + elif tid == implTag: + i = nextTree(buf, i) + elif tid == replayTag or tid == repConverterTag or tid == repDestroyTag or + tid == repWasMovedTag or tid == repCopyTag or tid == repSinkTag or + tid == repDupTag or tid == repTraceTag or tid == repDeepCopyTag or + tid == repEnumToStrTag or tid == repMethodTag or + tid == exportTag or tid == exportExceptTag or tid == includeTag: + let e = nextTree(buf, i) + hashRegion(s, c, buf, i, e) + i = e + elif tid == importTag: + let e = nextTree(buf, i) + hashRegion(s, c, buf, i, e) + for j in i ..< e: + if buf[j].kind == StringLit: + let suffix = pool.strings[buf[j].litId] + if suffix notin c.depSuffixes: c.depSuffixes.add suffix + i = e + else: + inc i # descend without hashing + else: + inc i + +proc icGroupSuffixes(config: ConfigRef): HashSet[string] = + ## Module suffixes of the --icGroup cycle members compiled by this very + ## process (their sidecars are being produced concurrently, so neither + ## chaining nor edge recording may depend on them). + result = initHashSet[string]() + for p in config.icGroup: + result.incl cachedModuleSuffix(config, fileInfoIdx(config, AbsoluteFile p)) + +proc writeCookieFile(config: ConfigRef; selfSuffix, tag, hex, ext: string) = + var dest = createTokenBuf(4) + dest.addParLe pool.tags.getOrIncl(tag), NoLineInfo + dest.addStrLit hex + dest.addParRi + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ext).string + writeFile(dest, path, OnlyIfChanged) + +proc writeIfaceCookie(config: ConfigRef; thisModule: int32; buf: TokenBuf): string = + let selfSuffix = modname(thisModule, config) + var c = CookieCtx(selfSuffix: selfSuffix) + # pre-pass: first (td ...) occurrence per type name, wherever it is embedded + var i = 0 + while i < buf.len: + if buf[i].kind == ParLe and buf[i].tagId == tdefTag and i+1 < buf.len and + buf[i+1].kind == SymbolDef: + let nm = pool.syms[buf[i+1].symId] + if not c.tdRanges.hasKey(nm): c.tdRanges[nm] = i + inc i + var s = newSha1State() + scanStmtsForCookie(s, c, buf) + # chain the direct deps' cookies; co-members of an --icGroup cycle are + # excluded (their sidecars are being produced by this very rule — chaining + # them would make the hash depend on within-group write order). + let groupSuffixes = icGroupSuffixes(config) + for dep in c.depSuffixes: + if dep == selfSuffix or dep in groupSuffixes: continue + let depIface = toGeneratedFile(config, AbsoluteFile(dep), ".iface.nif").string + s.update "|" + s.update dep + s.update ":" + s.update(try: readFile(depIface) except IOError, OSError: "") + result = $SecureHash(s.finalize()) + writeCookieFile(config, selfSuffix, "iface", result, ".iface.nif") + +proc writeImplCookie(config: ConfigRef; thisModule: int32; buf: TokenBuf; + ifaceHex: string) = + ## The implementation cookie: a line-info-free hash of the module's ENTIRE + ## serialized content (private defs and routine bodies included), with the + ## module's own iface cookie mixed in so impl sensitivity is a strict + ## superset of iface sensitivity (incl. the chained dep ifaces — a NeedsImpl + ## edge REPLACES the iface edge, it must not lose its triggers). Dependents + ## that consumed this module's bodies at compile time are gated on this file + ## instead of the iface cookie. Comment-only edits move neither cookie. + ## No id normalization here: a counter shift implies some real content + ## change elsewhere in the module, which flips the hash anyway — and any + ## body change is exactly what NeedsImpl dependents must see. + let selfSuffix = modname(thisModule, config) + var s = newSha1State() + for i in 0 ..< buf.len: + let t = buf[i] + if t.kind in {Symbol, SymbolDef}: + s.update(if t.kind == SymbolDef: " :" else: " ") + s.update pool.syms[t.symId] + else: + updateAtom s, t + s.update "|iface:" + s.update ifaceHex + writeCookieFile(config, selfSuffix, "impl", $SecureHash(s.finalize()), ".impl.nif") + +proc writeEdgesFile(config: ConfigRef; thisModule: int32; implDeps: seq[int]) = + ## Records which modules' bodies this compilation consumed at compile time + ## (`ModuleGraph.icImplDeps`): the NeedsImpl edge set. deps.nim reads this + ## sidecar when regenerating the build file and gates this module on those + ## dependencies' IMPL cookies instead of their iface cookies. + let selfSuffix = modname(thisModule, config) + let groupSuffixes = icGroupSuffixes(config) + var suffixes: seq[string] = @[] + for id in implDeps: + if id == thisModule.int: continue + let suffix = cachedModuleSuffix(config, FileIndex id) + if suffix.len == 0 or suffix == selfSuffix or suffix in groupSuffixes: + continue + if suffix notin suffixes: suffixes.add suffix + sort suffixes + var dest = createTokenBuf(4 + 2*suffixes.len) + dest.addParLe pool.tags.getOrIncl("edges"), NoLineInfo + for suffix in suffixes: + dest.addStrLit suffix + dest.addParRi + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".edges.nif").string + # Deliberately ALWAYS written (unlike every other output of the nim_m rule): + # nothing gates on this file's mtime — deps.nim only reads its content — so + # it doubles as the rule's freshness stamp. nifmake's `needsRebuild` takes + # the freshest output as proof of "ran since the inputs changed"; without an + # always-written output a rule whose re-run produces only content-identical + # (mtime-preserved) files would re-fire on every warm build (e.g. after an + # edit was reverted). Nimony's analog is its always-written `.s.nif`. + writeFile(dest, path) + proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; opsLog: seq[LogEntry]; - replayActions: seq[PNode] = @[]) = + replayActions: seq[PNode] = @[]; + implDeps: seq[int] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var content = createTokenBuf(300) @@ -864,6 +1236,10 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; # level, and the nifc backend can trust "semmed NIF older than the cnif # artifact" as an honest per-module unchanged stamp. writeFile(dest, d, OnlyIfChanged) + if not isDefined(config, "icNoIfaceGate"): + let ifaceHex = writeIfaceCookie(config, thisModule, dest) + writeImplCookie(config, thisModule, dest, ifaceHex) + writeEdgesFile(config, thisModule, implDeps) # --------------------------- Loader (lazy!) ----------------------------------------------- @@ -1850,3 +2226,4 @@ when isMainModule: echo obj.name, " ", obj.module, " ", obj.count let objb = parseSymName("abcdef.0121") echo objb.name, " ", objb.module, " ", objb.count + diff --git a/compiler/deps.nim b/compiler/deps.nim index dd69e096a8..857b3504b4 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -46,6 +46,43 @@ proc parsedFile(c: DepContext; f: FilePair): string = proc semmedFile(c: DepContext; f: FilePair): string = getNimcacheDir(c.config).string / f.modname & ".nif" +proc ifaceFile(c: DepContext; f: FilePair): string = + ## Interface-cookie sidecar written by `nim m` (ast2nif.writeIfaceCookie, + ## OnlyIfChanged). Dependents' nim_m rules use it as their input instead of + ## the semmed NIF: a body-only change in a dependency then keeps the sidecar + ## mtime and nifmake prunes the whole re-sem cascade behind it. + getNimcacheDir(c.config).string / f.modname & ".iface.nif" + +proc implFile(c: DepContext; suffix: string): string = + ## Implementation-cookie sidecar (ast2nif.writeImplCookie): flips on ANY + ## content change of the module (private bodies included; supersedes the + ## iface cookie). Used as the edge for dependents that consumed the + ## module's bodies at compile time (NeedsImpl edges). + getNimcacheDir(c.config).string / suffix & ".impl.nif" + +proc edgesFile(c: DepContext; f: FilePair): string = + getNimcacheDir(c.config).string / f.modname & ".edges.nif" + +proc readNeedsImpl(c: DepContext; f: FilePair): seq[string] = + ## Reads the module's recorded NeedsImpl edge set (module suffixes whose + ## bodies its last sem consumed at compile time). Missing file (never + ## compiled yet) -> empty: the rule fires anyway on the first build and the + ## recording exists from then on. Recordings are self-correcting with a + ## one-run lag: whatever changes a module's consumption set is itself a + ## gated input of its rule, so the rule re-fires and re-records. + result = @[] + if fileExists(c.edgesFile(f)): + var s = nifstreams.open(c.edgesFile(f)) + try: + discard processDirectives(s.r) + while true: + let t = next(s) + if t.kind == EofToken: break + if t.kind == StringLit: + result.add pool.strings[t.litId] + finally: + close s + proc findNifler(): string = # Look for nifler in common locations let nimDir = getAppDir() @@ -78,6 +115,27 @@ proc runNifler(c: DepContext; nimFile: string): bool = let cmd = quoteShell(c.nifler) & " deps " & quoteShell(nimFile) & " " & quoteShell(depsPath) let exitCode = execShellCmd(cmd) result = exitCode == 0 + if result: + # The build graph's `nifler parse --deps` rule outputs BOTH the parsed + # file and the deps file. Refreshing the deps file here would MASK that + # rule: nifmake's `needsRebuild` takes the freshest output as proof of + # "ran since the inputs changed", so the rule never re-fires and the + # parsed file goes stale. For an import-cycle group that loses the edit + # entirely — a non-representative member's source is not a direct input + # of the group's `nim_m` rule; its only build-graph connection is the + # (now stale) parsed file. Drop a genuinely stale parsed file so the + # nifler rule re-fires on the missing output. + let parsedPath = c.parsedFile(pair) + if fileExists(parsedPath) and + getLastModificationTime(parsedPath) < getLastModificationTime(nimFile): + removeFile(parsedPath) + # nifler writes OnlyIfChanged: after an edit that leaves the import set + # unchanged the deps file keeps its old mtime and would stay older than + # the source forever, re-running this scan (and re-deleting the parsed + # file) on every warm build. Bump it explicitly: it is the scan's own + # up-to-date marker. + if getLastModificationTime(depsPath) < getLastModificationTime(nimFile): + setLastModificationTime(depsPath, getTime()) proc resolveImport(c: DepContext; origin, toResolve: string): string = ## Resolve an import path using the compiler's normal module lookup rules. @@ -682,21 +740,78 @@ proc generateBuildFile(c: DepContext): string = b.addTree "input" b.addStrLit c.parsedFile(f) b.endTree() - # Depend only on the semmed files of dependencies *outside* this component. + # Depend on the dependencies *outside* this component — on their interface + # COOKIE sidecars, not the semmed NIFs themselves: the sidecar's mtime only + # moves when the dep's importer-visible surface (or, via hash chaining, any + # surface in its import closure) changed, so body-only edits stop the + # re-sem cascade right here. Dependencies whose BODIES the last sem of a + # member consumed at compile time (the recorded NeedsImpl edge set) are + # gated on their IMPL cookie instead, which flips on any content change: + # `const x = dep.foo()` then re-sems when foo's body changes. + # `-d:icNoIfaceGate` restores the old full-NIF edges. + let ifaceGate = not isDefined(c.config, "icNoIfaceGate") + var needsImpl = initHashSet[string]() + if ifaceGate: + # union over the members; restricted to the group's transitive dep + # closure: a stale recording naming a module this group no longer + # imports cannot be consumed anymore (and honoring it could even create + # a build-graph cycle after refactorings). + var reachable = initHashSet[string]() + var stack: seq[int] = @[] + for m in members: + for depIdx in c.nodes[m].deps: + if sccOf[depIdx] != sccOf[members[0]]: stack.add depIdx + var visited = initHashSet[int]() + while stack.len > 0: + let n = stack.pop() + if visited.containsOrIncl(n): continue + reachable.incl c.nodes[n].files[0].modname + for depIdx in c.nodes[n].deps: stack.add depIdx + for m in members: + for suffix in readNeedsImpl(c, c.nodes[m].files[0]): + if suffix in reachable: needsImpl.incl suffix var seenDep = initHashSet[string]() + var directDeps = initHashSet[string]() for m in members: for depIdx in c.nodes[m].deps: if sccOf[depIdx] == sccOf[m]: continue # intra-component edge - let depSem = c.semmedFile(c.nodes[depIdx].files[0]) - if not seenDep.containsOrIncl(depSem): + let depName = c.nodes[depIdx].files[0].modname + directDeps.incl depName + let depFile = + if not ifaceGate: c.semmedFile(c.nodes[depIdx].files[0]) + elif depName in needsImpl: c.implFile(depName) + else: c.ifaceFile(c.nodes[depIdx].files[0]) + if not seenDep.containsOrIncl(depFile): b.addTree "input" - b.addStrLit depSem + b.addStrLit depFile b.endTree() - # Output: one semmed NIF per member. + # NeedsImpl on modules that are not direct imports (bodies consumed via + # re-exports or transitively, e.g. a macro's private helper two hops + # away): additional impl-cookie inputs. + if ifaceGate: + var extra: seq[string] = @[] + for suffix in needsImpl: + if suffix notin directDeps: extra.add suffix + sort extra + for suffix in extra: + b.addTree "input" + b.addStrLit c.implFile(suffix) + b.endTree() + # Output: one semmed NIF (plus its cookie/edge sidecars) per member. for m in members: b.addTree "output" b.addStrLit c.semmedFile(c.nodes[m].files[0]) b.endTree() + if ifaceGate: + b.addTree "output" + b.addStrLit c.ifaceFile(c.nodes[m].files[0]) + b.endTree() + b.addTree "output" + b.addStrLit c.implFile(c.nodes[m].files[0].modname) + b.endTree() + b.addTree "output" + b.addStrLit c.edgesFile(c.nodes[m].files[0]) + b.endTree() b.endTree() # Final compilation step: generate executable from main module diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 2a36ba072b..46b9079d14 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -97,6 +97,18 @@ type # loads (reached only through system or demand-driven codegen) icFileReusedCnames*: HashSet[string] # their .c paths, so demand-created # BModules for them never write anything + icImplDeps*: IntSet # NeedsImpl edge tracking under `nim m`: + # module ids (FileIndex) whose routine BODIES + # this compilation consumed at compile time + # (VM-compiled or getImpl'ed). Written to the + # `.edges` sidecar; deps.nim then gates the + # dependent on those modules' IMPL cookie + # instead of the iface cookie, so e.g. + # `const x = dep.foo()` re-sems when foo's + # body changes. Bodies with inline semantics + # (templates/macros/generics/iterators) need + # no tracking: they are part of the iface + # cookie itself. packageSyms*: TStrTable deps*: IntSet # the dependency graph or potentially its transitive closure. @@ -690,6 +702,15 @@ proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} = ## into the demanding module instead. result = m.int notin g.icReusedModules +proc recordIcImplDep*(g: ModuleGraph; s: PSym) = + ## NeedsImpl edge tracking, see `icImplDeps`. Called from the compile-time + ## body consumption sites (vmgen's proc compilation, the getImpl opcodes). + ## Own-module and group-member entries are filtered out when the `.edges` + ## sidecar is written. + if g.config.cmd == cmdM and s != nil and s.kind in routineKinds and + s.itemId.module >= 0 and not isBackendMinted(s.itemId): + g.icImplDeps.incl module(s.itemId).int + proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) = diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 501b922b00..c4db260a22 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -15,7 +15,7 @@ import ../dist/checksums/src/checksums/sha1 when not defined(leanCompiler): import jsgen, docgen2 -import std/[syncio, objectdollar, assertions, tables, strutils, strtabs, sets] +import std/[syncio, objectdollar, assertions, tables, strutils, strtabs, sets, intsets] import renderer import ic/replayer @@ -264,7 +264,13 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator if m == module: replayActions.add n - writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, replayActions) + # NeedsImpl edge recording: which modules' bodies this process consumed + # at compile time (VM/getImpl). For an --icGroup cycle every member gets + # the union; intra-group entries are filtered by the writer. + var implDeps: seq[int] = @[] + for id in graph.icImplDeps: implDeps.add id + writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, + replayActions, implDeps) result = true diff --git a/compiler/vm.nim b/compiler/vm.nim index 0104a87e76..a2baa5ec3c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1310,6 +1310,9 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var a = regs[rb].node if a.kind == nkVarTy: a = a[0] if a.kind == nkSym: + # a macro observed this symbol's implementation: NeedsImpl edge to + # its home module under IC. + recordIcImplDep(c.graph, a.sym) regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit) else: copyTree(a.sym.ast) regs[ra].node.flags.incl nfIsRef @@ -1319,6 +1322,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeB(rkNode) let a = regs[rb].node if a.kind == nkSym: + recordIcImplDep(c.graph, a.sym) regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 1ac0d49424..ec5c6e78cb 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -36,7 +36,7 @@ import magicsys, options, lowerings, lineinfos, transf, astmsgs, treetab -from modulegraphs import getBody +from modulegraphs import getBody, recordIcImplDep when defined(nimCompilerStacktraceHints): import std/stackframes @@ -2460,6 +2460,10 @@ proc optimizeJumps(c: PCtx; start: int) = proc genProc(c: PCtx; s: PSym): VmProcInfo = result = c.procToCodePos.getOrDefault(s.id, NoVmProcInfo) if result.usedRegisters < 0: + # compile-time execution consumes this routine's BODY: under IC that is a + # NeedsImpl dependency on the routine's home module (iface-cookie gating + # alone would miss body-only edits, e.g. `const x = dep.foo()`). + recordIcImplDep(c.graph, s) #if s.name.s == "outterMacro" or s.name.s == "innerProc": # echo "GENERATING CODE FOR ", s.name.s let last = c.code.len-1