From 1dc079b7235655f8135600c9a9b3d13a30bef02e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 9 Jul 2026 22:13:39 +0200 Subject: [PATCH] nim-track: make include files work (#25977) --- compiler/ast2nif.nim | 39 +++++++++++++- compiler/commands.nim | 12 ++++- compiler/deps.nim | 92 ++++++++++++++++++++++++++++++++ compiler/icconfig.nim | 23 +++++--- compiler/idetools.nim | 118 +++++++++++++++++++++++++++++++----------- compiler/options.nim | 2 +- koch.nim | 4 +- 7 files changed, 246 insertions(+), 44 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index bda1296f99..d8015285fc 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -962,6 +962,34 @@ proc emitSigOccurrences(w: var Writer; n: PNode) = else: for i in 0 ..< n.safeLen: emitSigOccurrences(w, n[i]) +proc emitFwdDecl(w: var Writer; n: PNode; sym: PSym) = + ## A routine's forward declaration (`proc foo(...)` with no body, later followed + ## by `proc foo(...) = ...`) is a distinct top-level node, but the routine has a + ## SINGLE `sdef`, emitted at the IMPLEMENTATION site (`sym.infoImpl`) — so the + ## prototype's own position would otherwise vanish from the `.bif`. Tee it into + ## the `deps` side-channel as a POSITIONED `(sig @proto )`: the loader + ## skips the `sig` tag (processTopLevel), but `idetools.scanDef` finds the + ## `SymbolDef` and reports the enclosing tag's line info — so a `--def` on a + ## forward-declared proc returns TWO results (prototype + implementation), which + ## is desired. Safe against symbol resolution: the loader rebuilds its name->pos + ## table from the CONTENT body (`buildPosIndex`, written after `deps`, last write + ## wins) so the real `sdef` still resolves; the extra on-disk index entry has no + ## resolution consumer. The prototype's signature symbols (param names and the + ## symbols in their type expressions) are teed too, positioned at the prototype, + ## exactly as `emitSigOccurrences` records them for the implementation. + # The `SymbolDef` carries the prototype line info too (not just the enclosing + # tag): `scanDef` reads the position from the tag, but pass-1 `findPos` matches + # a token by its OWN line info, so this is what makes a query issued AT the + # prototype position resolve the symbol. + let protoInfo = trLineInfo(w, n[namePos].info) + let sid = pool.syms.getOrIncl(w.toNifSymName(sym)) + w.deps.addParLe sigTag, protoInfo + w.deps.addSymDef sid, protoInfo # scanDef reports this as a def + w.deps.addSymUse sid, protoInfo # findPos (pass 1) / scanUses match a Symbol use + w.deps.addParRi + if sfFromGeneric notin sym.flagsImpl and paramsPos < n.safeLen: + emitSigOccurrences(w, n[paramsPos]) + proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) = if n == nil: dest.addDotToken @@ -1035,7 +1063,16 @@ proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) = # For top-level named routines (not forAst), just write the symbol. # The full AST will be stored in the symbol's sdef. if not forAst and n[namePos].kind == nkSym: - writeSym(w, dest, n[namePos].sym) + let s = n[namePos].sym + writeSym(w, dest, s) + # A forward declaration is a SECOND top-level node for `s` (body-less here; + # the real body — and the lone sdef — lands at the implementation). Tee the + # prototype's own position so goto-def / find-usages surface it as well. + let impl = s.astImpl + if n.safeLen > bodyPos and n[bodyPos].kind == nkEmpty and + impl != nil and impl != n and + impl.safeLen > bodyPos and impl[bodyPos].kind != nkEmpty: + emitFwdDecl(w, n, s) else: # Writing AST inside sdef or anonymous proc: write full structure inc w.inProc diff --git a/compiler/commands.nim b/compiler/commands.nim index d7f0ad735b..ccf57142da 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1325,8 +1325,16 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser; # support UNIX style filenames everywhere for portable build scripts: if config.projectName.len == 0: config.projectName = unixToNativePath(p.key) - config.arguments = cmdLineRest(p) - result = true + if config.cmd == cmdTrack: + # `nim track PROJ --def:...`: unlike a normal command (where everything + # after the project file is passed to the compiled program), `track` + # accepts its IDE-query switches AFTER the project — the natural, + # nimsuggest-like invocation form. So don't swallow the rest of the line + # into `arguments`; keep parsing the remaining tokens as switches. + result = false + else: + config.arguments = cmdLineRest(p) + result = true else: result = false inc argsCount diff --git a/compiler/deps.nim b/compiler/deps.nim index 44dfa007d9..63ef12a2b1 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -590,6 +590,98 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) = elif t.kind == ParRi: dec depth t = next(s) +proc collectIncludeNames(depsPath: string; names: var seq[string]) = + ## Lightweight scan of a `.deps.nif` prelude: collect the raw path text of + ## every entry inside an `(include ...)` node (idents like `semexprs`, string + ## literals like `"system/mmdisp"`, and the leaves of `a/b` path infixes). + ## Liberal by design — it also picks up entries under a statically-false + ## `(when ...)`; that is harmless for the only caller (`includerSbifs`), whose + ## over-collection just costs an extra, result-free bif scan downstream. + if not fileExists(depsPath): return + var s = nifstreams.open(depsPath) + defer: nifstreams.close(s) + discard processDirectives(s.r) + var depth = 0 + var includeDepth = 0 # the `depth` at which the current `(include` opened; 0 = not inside one + var t = next(s) + while t.kind != EofToken: + case t.kind + of ParLe: + inc depth + if includeDepth == 0 and pool.tags[t.tagId] == "include": + includeDepth = depth + of ParRi: + if includeDepth != 0 and depth == includeDepth: + includeDepth = 0 + dec depth + of Ident, StringLit: + if includeDepth != 0: + names.add pool.strings[t.litId] + else: discard + t = next(s) + +proc entryStemBase(roots: seq[string]; name: string): (string, string) = + ## Resolve include entry `name` to (deps-stem, base-name); ("","") if unfound. + for r in roots: + let p = r / name.addFileExt("nim") + if fileExists(p): + return (moduleSuffix(p, []), splitFile(p).name) + result = ("", "") + +proc includerSbifs*(conf: ConfigRef; targetFile: AbsoluteFile): seq[string] = + ## For an include file `targetFile`, return the `.s.bif` paths of every module + ## that includes it — directly OR transitively (following the include chain + ## `module -> incA -> incB -> targetFile`). `nim track` uses this to avoid + ## loading and scanning every module bif: an include file has no bif of its + ## own, so its type-checked tokens live in the *including* module's bif. Only + ## the small `.deps.nif` preludes are read here, never a `.s.bif`. + const depsExt = ".deps.nif" + let nc = getNimcacheDir(conf).string + + # Candidate roots for resolving an `(include X)` entry to a real file, so its + # module suffix (== its own deps-file stem) can be computed. Include entries + # carry any sub-path (`system/mmdisp`), so the file's *directory* roots suffice: + # the target's own dir, the project dir, and the search paths cover the + # compiler, the stdlib and typical single-tree projects. + var roots: seq[string] = @[parentDir(targetFile.string)] + if conf.projectPath.string.len > 0: roots.add conf.projectPath.string + for sp in conf.searchPaths: roots.add sp.string + + # One pass over every prelude builds the reverse include graph, keyed by base + # file name: `includedBy[b]` = deps stems whose owner directly `include`s a + # file named `b`. `stemBase` maps an include-only file's deps stem back to its + # own base name, so the walk can climb through nested includes. + var includedBy = initTable[string, seq[string]]() + var stemBase = initTable[string, string]() + for depsPath in walkFiles(nc / "*" & depsExt): + let base = extractFilename(depsPath) + if base.endsWith(".p" & depsExt): continue # `.p.deps.nif` twin + let ownerStem = base[0 ..< base.len - depsExt.len] + var names: seq[string] = @[] + collectIncludeNames(depsPath, names) + for n in names: + let (childStem, childBase) = entryStemBase(roots, n) + if childBase.len == 0: continue + includedBy.mgetOrPut(childBase, @[]).add ownerStem + stemBase[childStem] = childBase # this child's stem -> its base name + + # Walk UP from the target: a deps stem that includes the current base name is + # either a module (has a `.s.bif` -> collect it) or itself an include file + # (recurse via its own base name). + result = @[] + var seenBase = initHashSet[string]() + var work = @[splitFile(targetFile.string).name] + while work.len > 0: + let b = work.pop() + if seenBase.containsOrIncl(b): continue + for stem in includedBy.getOrDefault(b): + let sbif = nc / stem & ".s.bif" + if fileExists(sbif): + if sbif notin result: result.add sbif # module owner + else: + let ob = stemBase.getOrDefault(stem) # include-only owner: climb higher + if ob.len > 0: work.add ob + proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) = ## Process a module: run nifler and read deps if not runNifler(c, pair.nimFile): diff --git a/compiler/icconfig.nim b/compiler/icconfig.nim index 0250ee0583..a0255c23dc 100644 --- a/compiler/icconfig.nim +++ b/compiler/icconfig.nim @@ -260,17 +260,26 @@ proc ensureIcConfig*(conf: ConfigRef) = if not fileExists(outPath) or sourcesChanged(outPath): createDir(cacheDir) # Re-invoke ourselves as the config producer: reuse this process's command - # line, dropping the command argument (`ic`) in favour of `icconfig` and the - # explicit output path, both BEFORE the project file (anything after the - # project is swallowed into `config.arguments` by `cmdLineRest`). The - # producer re-reads `nim.cfg` itself. + # line, dropping the command argument (`ic`/`track`) in favour of `icconfig` + # and the explicit output path. Every switch must land BEFORE the project + # file, because anything after the project is swallowed into + # `config.arguments` by `cmdLineRest` (and a non-empty `arguments` without + # `--run` is a hard error). Callers may legitimately put switches after the + # project — `nim track PROJ --def:...` — so we re-order rather than replay + # verbatim: all `-`-prefixed switches first (in encounter order), then the + # non-switch project token(s). The producer re-reads `nim.cfg` itself. var pargs = @["icconfig", "--icConfigOut:" & outPath] + var rest: seq[string] = @[] var droppedCmd = false for a in commandLineParams(): - if not droppedCmd and a.len > 0 and a[0] != '-': - droppedCmd = true # drop the original command token (`ic`) - else: + if a.len == 0: continue + if a[0] == '-': pargs.add a + elif not droppedCmd: + droppedCmd = true # drop the original command token (`ic`/`track`) + else: + rest.add a # project file (and any further non-switch tokens) go last + for a in rest: pargs.add a let p = startProcess(getAppFilename(), args = pargs, options = {poStdErrToStdOut}) let outp = p.outputStream.readAll() diff --git a/compiler/idetools.nim b/compiler/idetools.nim index 05266966e8..39be60f98a 100644 --- a/compiler/idetools.nim +++ b/compiler/idetools.nim @@ -35,8 +35,9 @@ import std / [os, strutils, sets] import options, msgs, pathutils import lineinfos as astli import ast2nif # toNifFilename +from deps import includerSbifs # deps-guided include-file lookup import "../dist/nimony/src/lib/nifcore" -from "../dist/nimony/src/lib" / bif import load, BifModule +from "../dist/nimony/src/lib" / bif import load, BifModule, containsSym proc identLen(name: string): int = ## Length of the displayed identifier: the run before the first `.` of a @@ -122,21 +123,37 @@ proc emit(conf: ConfigRef; c: Cursor; section: IdeCmd; name: string; else: conf.suggestWriteln(formatSuggest(s)) -proc scanUses(conf: ConfigRef; m: var BifModule; targetName: string; +proc tokenSymId(c: Cursor): SymId {.inline.} = + ## SymId (in the cursor's own per-file pool) of a `Symbol`/`SymbolDef` token, + ## or `SymId(0)` for an inline-encoded one — which is never our search target: + ## a mangled name (`ident.disamb.suffix`) is always longer than + ## `StrInlineMaxLen`, so every occurrence of the symbol we look for is stored by + ## pool id, decoded here with a shift and no string materialization. + if isInlineLit(c): SymId(0) else: SymId(combinedPayload(c) shr 1) + +template symMatches(c: Cursor): bool = + ## True when the token at `c` is the searched symbol. The fast path is a pure + ## integer compare against `targetSym` (the symbol's id in THIS module's pool, + ## resolved once per file by the caller). `targetSym == 0` means the name is not + ## representable as a pool id (a rare <=3-byte local): fall back to a string + ## compare, correct for both inline and pooled encodings. + (if targetSym != SymId(0): tokenSymId(c) == targetSym else: symName(c) == targetName) + +proc scanUses(conf: ConfigRef; m: var BifModule; targetSym: SymId; targetName: string; seen: var HashSet[string]) = ## `--usages`: report every `Symbol` (use) occurrence with valid line info. if m.buf.len == 0: return var c = m.buf.beginRead() while c.hasMore: - if c.kind == Symbol and symName(c) == targetName and rawLineInfo(c).isValid: + if c.kind == Symbol and symMatches(c) and rawLineInfo(c).isValid: emit(conf, c, ideUse, targetName, seen) inc c c.endRead() -proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string; +proc scanDef(conf: ConfigRef; m: var BifModule; targetSym: SymId; targetName: string; seen: var HashSet[string]) = - ## `--def`: report the declaration of `targetName` if this module owns it (has - ## its `SymbolDef`). The `SymbolDef` token itself carries no line info; the + ## `--def`: report the declaration of the target symbol if this module owns it + ## (has its `SymbolDef`). The `SymbolDef` token itself carries no line info; the ## declaration location lives on the *enclosing tag* (e.g. `(sd @file:line:col`, ## like `bif.buildIndex`'s `mostRecentTagPos`). When that tag has no line info ## either, fall back to the declaration-site `Symbol` occurrence — but only in @@ -153,7 +170,7 @@ proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string; mostRecentTagPos = cursorToPosition(m.buf, c) inc c of SymbolDef: - if symName(c) == targetName: + if symMatches(c): sawDef = true var tc = cursorAt(m.buf, mostRecentTagPos) if rawLineInfo(tc).isValid: @@ -162,7 +179,7 @@ proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string; tc.endRead() inc c of Symbol: - if fallbackPos < 0 and symName(c) == targetName and rawLineInfo(c).isValid: + if fallbackPos < 0 and symMatches(c) and rawLineInfo(c).isValid: fallbackPos = cursorToPosition(m.buf, c) inc c else: @@ -173,14 +190,32 @@ proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string; emit(conf, fc, ideDef, targetName, seen) fc.endRead() -proc scanBuf(conf: ConfigRef; m: var BifModule; section: IdeCmd; targetName: string; - seen: var HashSet[string]) = - ## Emit hits for `targetName` in `m` per the query kind. `ideDus` +proc scanBuf(conf: ConfigRef; m: var BifModule; section: IdeCmd; + targetSym: SymId; targetName: string; seen: var HashSet[string]) = + ## Emit hits for the target symbol in `m` per the query kind. `ideDus` ## (`--defusages`) reports both the definition and every usage. if section in {ideDef, ideDus}: - scanDef(conf, m, targetName, seen) + scanDef(conf, m, targetSym, targetName, seen) if section in {ideUse, ideDus}: - scanUses(conf, m, targetName, seen) + scanUses(conf, m, targetSym, targetName, seen) + +proc findPos(conf: ConfigRef; m: var BifModule; target: TLineInfo; + foundName: var string): bool = + ## Scan `m` for the `Symbol`/`SymbolDef` token covering the queried position + ## `target` and set `foundName` to its mangled name. Returns true on a hit. + if m.buf.len == 0: return false + var c = m.buf.beginRead() + result = false + while c.hasMore: + let k = c.kind + if k == Symbol or k == SymbolDef: + let nm = symName(c) + if posMatch(c, conf, target, identLen(nm)): + foundName = nm + result = true + break + inc c + c.endRead() proc runIdeQuery*(conf: ConfigRef) = ## Entry point: called from `main.nim` after `commandCheck` when a @@ -191,33 +226,54 @@ proc runIdeQuery*(conf: ConfigRef) = let target = conf.m.trackPos if target.fileIndex.int32 < 0: return - # Pass 1: position -> symbol, in the queried module's own .s.bif. + # Pass 1: position -> symbol. Try the queried file's own module bif first (the + # fast path when the position is inside a real module). An include file has no + # module bif of its own — its tokens live in the *including* module's bif with + # include-file line info — so when the direct lookup misses, consult the + # `.deps.nif` preludes (`includerSbifs`) to load only the module(s) that + # include the queried file (directly or transitively), never every bif in the + # nimcache. `ownerFile` is the bif that owns the hit. let modFile = toNifFilename(conf, target.fileIndex) - if not fileExists(modFile): return - var qm = load(modFile) var foundName = "" - block find: - if qm.buf.len == 0: break find - var c = qm.buf.beginRead() - while c.hasMore: - let k = c.kind - if k == Symbol or k == SymbolDef: - let nm = symName(c) - if posMatch(c, conf, target, identLen(nm)): - foundName = nm - break find - inc c - c.endRead() + var ownerFile = "" + if fileExists(modFile): + var qm = load(modFile) + if findPos(conf, qm, target, foundName): + ownerFile = modFile + if foundName.len == 0: + for cand in includerSbifs(conf, toFullPath(conf, target.fileIndex).AbsoluteFile): + if cand == modFile: continue + var m = load(cand) + if findPos(conf, m, target, foundName): + ownerFile = cand + break if foundName.len == 0: return # Pass 2: emit definition / usages. `seen` spans every module so a location is # reported once even when scanned across the whole nimcache. + # + # Cross-file matching is by SymId, not by decoding every token's name. Two + # filters keep it cheap: + # 1. `bif.containsSym` — a sym-table-only probe that reads just the small + # trailing pools, NOT the token block or any `BiTable`. A module that never + # references the symbol is rejected here without a full `load` (no pools + # built, no token block mapped) — so a query whose symbol lives in a few + # modules no longer pays to load the whole nimcache. + # 2. For a module that does contain it, `bif.load` mints a fresh per-file pool, + # so the name is resolved to THIS file's SymId once via `getKeyId`; the scan + # then compares integer ids per token instead of materializing a string for + # each (see `symMatches`). var seen = initHashSet[string]() if isGlobalName(foundName): for f in walkFiles((getNimcacheDir(conf).string) / "*.s.bif"): + if not containsSym(f, foundName): continue var m = load(f) - scanBuf(conf, m, section, foundName, seen) + let tid = m.buf.pool.syms.getKeyId(foundName) + if tid != SymId(0): + scanBuf(conf, m, section, tid, foundName, seen) else: # Local symbol: its mangled name is not unique across modules, so restrict - # the scan to the module it lives in (the queried module). - scanBuf(conf, qm, section, foundName, seen) + # the scan to the module it lives in (the one that owns the queried position). + var qm = load(ownerFile) + let tid = qm.buf.pool.syms.getKeyId(foundName) + scanBuf(conf, qm, section, tid, foundName, seen) diff --git a/compiler/options.nim b/compiler/options.nim index 283f7e04c3..10ff8b9d88 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -29,7 +29,7 @@ const nimEnableCovariance* = defined(nimEnableCovariance) - icFormatVersion* = "29" + icFormatVersion* = "30" ## Version of the IC cache format (the sem-NIF module layout written by ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever ## that layout changes: `commandIc` wipes a nimcache whose `ic.version` diff --git a/koch.nim b/koch.nim index 4364cfa254..9d63618e67 100644 --- a/koch.nim +++ b/koch.nim @@ -16,11 +16,11 @@ const ChecksumsStableCommit = "5c132cd332cce5d64a0da9ac3e4c9664313dccb4" # 0.2.2 SatStableCommit = "9d52513b3c68bfb929dbd687d4fb2836cfee6936" - NimonyStableCommit = "6f9ac6655dc6724ae4e5ccb93b8123c18d54391a" # unversioned \ + NimonyStableCommit = "863a04c07277eaa0a6bb2a0059d731a59cbc09b4" # unversioned \ # Note that Nimony uses Nim as a git submodule but we don't want to install # Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive # is **required** here. - # Commit from 2026-07-03 -- .bif files are memory mapped too + # Commit from 2026-07-09 -- .bif files are memory mapped too # examples of possible values for fusion: #head, #ea82b54, 1.2.3 FusionStableHash = "#562467452b32cb7a97410ea177f083e6d8405734"