From eea4721296f54fb430a609cdd40700d4f0c7fa38 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 9 Jul 2026 12:07:03 +0200 Subject: [PATCH] nim-track: make include files work --- compiler/commands.nim | 12 +++++- compiler/deps.nim | 92 +++++++++++++++++++++++++++++++++++++++++++ compiler/icconfig.nim | 23 +++++++---- compiler/idetools.nim | 56 ++++++++++++++++++-------- 4 files changed, 158 insertions(+), 25 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index d7f0ad735b..ccf57142da 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1325,8 +1325,16 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser; # support UNIX style filenames everywhere for portable build scripts: if config.projectName.len == 0: config.projectName = unixToNativePath(p.key) - config.arguments = cmdLineRest(p) - result = true + if config.cmd == cmdTrack: + # `nim track PROJ --def:...`: unlike a normal command (where everything + # after the project file is passed to the compiled program), `track` + # accepts its IDE-query switches AFTER the project — the natural, + # nimsuggest-like invocation form. So don't swallow the rest of the line + # into `arguments`; keep parsing the remaining tokens as switches. + result = false + else: + config.arguments = cmdLineRest(p) + result = true else: result = false inc argsCount diff --git a/compiler/deps.nim b/compiler/deps.nim index 44dfa007d9..63ef12a2b1 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -590,6 +590,98 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) = elif t.kind == ParRi: dec depth t = next(s) +proc collectIncludeNames(depsPath: string; names: var seq[string]) = + ## Lightweight scan of a `.deps.nif` prelude: collect the raw path text of + ## every entry inside an `(include ...)` node (idents like `semexprs`, string + ## literals like `"system/mmdisp"`, and the leaves of `a/b` path infixes). + ## Liberal by design — it also picks up entries under a statically-false + ## `(when ...)`; that is harmless for the only caller (`includerSbifs`), whose + ## over-collection just costs an extra, result-free bif scan downstream. + if not fileExists(depsPath): return + var s = nifstreams.open(depsPath) + defer: nifstreams.close(s) + discard processDirectives(s.r) + var depth = 0 + var includeDepth = 0 # the `depth` at which the current `(include` opened; 0 = not inside one + var t = next(s) + while t.kind != EofToken: + case t.kind + of ParLe: + inc depth + if includeDepth == 0 and pool.tags[t.tagId] == "include": + includeDepth = depth + of ParRi: + if includeDepth != 0 and depth == includeDepth: + includeDepth = 0 + dec depth + of Ident, StringLit: + if includeDepth != 0: + names.add pool.strings[t.litId] + else: discard + t = next(s) + +proc entryStemBase(roots: seq[string]; name: string): (string, string) = + ## Resolve include entry `name` to (deps-stem, base-name); ("","") if unfound. + for r in roots: + let p = r / name.addFileExt("nim") + if fileExists(p): + return (moduleSuffix(p, []), splitFile(p).name) + result = ("", "") + +proc includerSbifs*(conf: ConfigRef; targetFile: AbsoluteFile): seq[string] = + ## For an include file `targetFile`, return the `.s.bif` paths of every module + ## that includes it — directly OR transitively (following the include chain + ## `module -> incA -> incB -> targetFile`). `nim track` uses this to avoid + ## loading and scanning every module bif: an include file has no bif of its + ## own, so its type-checked tokens live in the *including* module's bif. Only + ## the small `.deps.nif` preludes are read here, never a `.s.bif`. + const depsExt = ".deps.nif" + let nc = getNimcacheDir(conf).string + + # Candidate roots for resolving an `(include X)` entry to a real file, so its + # module suffix (== its own deps-file stem) can be computed. Include entries + # carry any sub-path (`system/mmdisp`), so the file's *directory* roots suffice: + # the target's own dir, the project dir, and the search paths cover the + # compiler, the stdlib and typical single-tree projects. + var roots: seq[string] = @[parentDir(targetFile.string)] + if conf.projectPath.string.len > 0: roots.add conf.projectPath.string + for sp in conf.searchPaths: roots.add sp.string + + # One pass over every prelude builds the reverse include graph, keyed by base + # file name: `includedBy[b]` = deps stems whose owner directly `include`s a + # file named `b`. `stemBase` maps an include-only file's deps stem back to its + # own base name, so the walk can climb through nested includes. + var includedBy = initTable[string, seq[string]]() + var stemBase = initTable[string, string]() + for depsPath in walkFiles(nc / "*" & depsExt): + let base = extractFilename(depsPath) + if base.endsWith(".p" & depsExt): continue # `.p.deps.nif` twin + let ownerStem = base[0 ..< base.len - depsExt.len] + var names: seq[string] = @[] + collectIncludeNames(depsPath, names) + for n in names: + let (childStem, childBase) = entryStemBase(roots, n) + if childBase.len == 0: continue + includedBy.mgetOrPut(childBase, @[]).add ownerStem + stemBase[childStem] = childBase # this child's stem -> its base name + + # Walk UP from the target: a deps stem that includes the current base name is + # either a module (has a `.s.bif` -> collect it) or itself an include file + # (recurse via its own base name). + result = @[] + var seenBase = initHashSet[string]() + var work = @[splitFile(targetFile.string).name] + while work.len > 0: + let b = work.pop() + if seenBase.containsOrIncl(b): continue + for stem in includedBy.getOrDefault(b): + let sbif = nc / stem & ".s.bif" + if fileExists(sbif): + if sbif notin result: result.add sbif # module owner + else: + let ob = stemBase.getOrDefault(stem) # include-only owner: climb higher + if ob.len > 0: work.add ob + proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) = ## Process a module: run nifler and read deps if not runNifler(c, pair.nimFile): diff --git a/compiler/icconfig.nim b/compiler/icconfig.nim index 0250ee0583..a0255c23dc 100644 --- a/compiler/icconfig.nim +++ b/compiler/icconfig.nim @@ -260,17 +260,26 @@ proc ensureIcConfig*(conf: ConfigRef) = if not fileExists(outPath) or sourcesChanged(outPath): createDir(cacheDir) # Re-invoke ourselves as the config producer: reuse this process's command - # line, dropping the command argument (`ic`) in favour of `icconfig` and the - # explicit output path, both BEFORE the project file (anything after the - # project is swallowed into `config.arguments` by `cmdLineRest`). The - # producer re-reads `nim.cfg` itself. + # line, dropping the command argument (`ic`/`track`) in favour of `icconfig` + # and the explicit output path. Every switch must land BEFORE the project + # file, because anything after the project is swallowed into + # `config.arguments` by `cmdLineRest` (and a non-empty `arguments` without + # `--run` is a hard error). Callers may legitimately put switches after the + # project — `nim track PROJ --def:...` — so we re-order rather than replay + # verbatim: all `-`-prefixed switches first (in encounter order), then the + # non-switch project token(s). The producer re-reads `nim.cfg` itself. var pargs = @["icconfig", "--icConfigOut:" & outPath] + var rest: seq[string] = @[] var droppedCmd = false for a in commandLineParams(): - if not droppedCmd and a.len > 0 and a[0] != '-': - droppedCmd = true # drop the original command token (`ic`) - else: + if a.len == 0: continue + if a[0] == '-': pargs.add a + elif not droppedCmd: + droppedCmd = true # drop the original command token (`ic`/`track`) + else: + rest.add a # project file (and any further non-switch tokens) go last + for a in rest: pargs.add a let p = startProcess(getAppFilename(), args = pargs, options = {poStdErrToStdOut}) let outp = p.outputStream.readAll() diff --git a/compiler/idetools.nim b/compiler/idetools.nim index 05266966e8..9262cce81e 100644 --- a/compiler/idetools.nim +++ b/compiler/idetools.nim @@ -35,6 +35,7 @@ import std / [os, strutils, sets] import options, msgs, pathutils import lineinfos as astli import ast2nif # toNifFilename +from deps import includerSbifs # deps-guided include-file lookup import "../dist/nimony/src/lib/nifcore" from "../dist/nimony/src/lib" / bif import load, BifModule @@ -182,6 +183,24 @@ proc scanBuf(conf: ConfigRef; m: var BifModule; section: IdeCmd; targetName: str if section in {ideUse, ideDus}: scanUses(conf, m, targetName, seen) +proc findPos(conf: ConfigRef; m: var BifModule; target: TLineInfo; + foundName: var string): bool = + ## Scan `m` for the `Symbol`/`SymbolDef` token covering the queried position + ## `target` and set `foundName` to its mangled name. Returns true on a hit. + if m.buf.len == 0: return false + var c = m.buf.beginRead() + result = false + while c.hasMore: + let k = c.kind + if k == Symbol or k == SymbolDef: + let nm = symName(c) + if posMatch(c, conf, target, identLen(nm)): + foundName = nm + result = true + break + inc c + c.endRead() + proc runIdeQuery*(conf: ConfigRef) = ## Entry point: called from `main.nim` after `commandCheck` when a ## `--def`/`--usages` query is active. Assumes the check just emitted the @@ -191,23 +210,27 @@ proc runIdeQuery*(conf: ConfigRef) = let target = conf.m.trackPos if target.fileIndex.int32 < 0: return - # Pass 1: position -> symbol, in the queried module's own .s.bif. + # Pass 1: position -> symbol. Try the queried file's own module bif first (the + # fast path when the position is inside a real module). An include file has no + # module bif of its own — its tokens live in the *including* module's bif with + # include-file line info — so when the direct lookup misses, consult the + # `.deps.nif` preludes (`includerSbifs`) to load only the module(s) that + # include the queried file (directly or transitively), never every bif in the + # nimcache. `ownerFile` is the bif that owns the hit. let modFile = toNifFilename(conf, target.fileIndex) - if not fileExists(modFile): return - var qm = load(modFile) var foundName = "" - block find: - if qm.buf.len == 0: break find - var c = qm.buf.beginRead() - while c.hasMore: - let k = c.kind - if k == Symbol or k == SymbolDef: - let nm = symName(c) - if posMatch(c, conf, target, identLen(nm)): - foundName = nm - break find - inc c - c.endRead() + var ownerFile = "" + if fileExists(modFile): + var qm = load(modFile) + if findPos(conf, qm, target, foundName): + ownerFile = modFile + if foundName.len == 0: + for cand in includerSbifs(conf, toFullPath(conf, target.fileIndex).AbsoluteFile): + if cand == modFile: continue + var m = load(cand) + if findPos(conf, m, target, foundName): + ownerFile = cand + break if foundName.len == 0: return # Pass 2: emit definition / usages. `seen` spans every module so a location is @@ -219,5 +242,6 @@ proc runIdeQuery*(conf: ConfigRef) = scanBuf(conf, m, section, foundName, seen) else: # Local symbol: its mangled name is not unique across modules, so restrict - # the scan to the module it lives in (the queried module). + # the scan to the module it lives in (the one that owns the queried position). + var qm = load(ownerFile) scanBuf(conf, qm, section, foundName, seen)