nim-track: make include files work (#25977)

compiler/ast2nif.nim

@@ -962,6 +962,34 @@ proc emitSigOccurrences(w: var Writer; n: PNode) =
   else:
     for i in 0 ..< n.safeLen: emitSigOccurrences(w, n[i])
 
+proc emitFwdDecl(w: var Writer; n: PNode; sym: PSym) =
+  ## A routine's forward declaration (`proc foo(...)` with no body, later followed
+  ## by `proc foo(...) = ...`) is a distinct top-level node, but the routine has a
+  ## SINGLE `sdef`, emitted at the IMPLEMENTATION site (`sym.infoImpl`) — so the
+  ## prototype's own position would otherwise vanish from the `.bif`. Tee it into
+  ## the `deps` side-channel as a POSITIONED `(sig @proto <symDef>)`: the loader
+  ## skips the `sig` tag (processTopLevel), but `idetools.scanDef` finds the
+  ## `SymbolDef` and reports the enclosing tag's line info — so a `--def` on a
+  ## forward-declared proc returns TWO results (prototype + implementation), which
+  ## is desired. Safe against symbol resolution: the loader rebuilds its name->pos
+  ## table from the CONTENT body (`buildPosIndex`, written after `deps`, last write
+  ## wins) so the real `sdef` still resolves; the extra on-disk index entry has no
+  ## resolution consumer. The prototype's signature symbols (param names and the
+  ## symbols in their type expressions) are teed too, positioned at the prototype,
+  ## exactly as `emitSigOccurrences` records them for the implementation.
+  # The `SymbolDef` carries the prototype line info too (not just the enclosing
+  # tag): `scanDef` reads the position from the tag, but pass-1 `findPos` matches
+  # a token by its OWN line info, so this is what makes a query issued AT the
+  # prototype position resolve the symbol.
+  let protoInfo = trLineInfo(w, n[namePos].info)
+  let sid = pool.syms.getOrIncl(w.toNifSymName(sym))
+  w.deps.addParLe sigTag, protoInfo
+  w.deps.addSymDef sid, protoInfo   # scanDef reports this as a def
+  w.deps.addSymUse sid, protoInfo   # findPos (pass 1) / scanUses match a Symbol use
+  w.deps.addParRi
+  if sfFromGeneric notin sym.flagsImpl and paramsPos < n.safeLen:
+    emitSigOccurrences(w, n[paramsPos])
+
 proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) =
   if n == nil:
     dest.addDotToken
@@ -1035,7 +1063,16 @@ proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) =
       # For top-level named routines (not forAst), just write the symbol.
       # The full AST will be stored in the symbol's sdef.
       if not forAst and n[namePos].kind == nkSym:
-        writeSym(w, dest, n[namePos].sym)
+        let s = n[namePos].sym
+        writeSym(w, dest, s)
+        # A forward declaration is a SECOND top-level node for `s` (body-less here;
+        # the real body — and the lone sdef — lands at the implementation). Tee the
+        # prototype's own position so goto-def / find-usages surface it as well.
+        let impl = s.astImpl
+        if n.safeLen > bodyPos and n[bodyPos].kind == nkEmpty and
+           impl != nil and impl != n and
+           impl.safeLen > bodyPos and impl[bodyPos].kind != nkEmpty:
+          emitFwdDecl(w, n, s)
       else:
         # Writing AST inside sdef or anonymous proc: write full structure
         inc w.inProc

compiler/commands.nim

@@ -1325,8 +1325,16 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
       # support UNIX style filenames everywhere for portable build scripts:
       if config.projectName.len == 0:
         config.projectName = unixToNativePath(p.key)
-      config.arguments = cmdLineRest(p)
-      result = true
+      if config.cmd == cmdTrack:
+        # `nim track PROJ --def:...`: unlike a normal command (where everything
+        # after the project file is passed to the compiled program), `track`
+        # accepts its IDE-query switches AFTER the project — the natural,
+        # nimsuggest-like invocation form. So don't swallow the rest of the line
+        # into `arguments`; keep parsing the remaining tokens as switches.
+        result = false
+      else:
+        config.arguments = cmdLineRest(p)
+        result = true
     else:
       result = false
   inc argsCount

compiler/deps.nim

@@ -590,6 +590,98 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) =
           elif t.kind == ParRi: dec depth
     t = next(s)
 
+proc collectIncludeNames(depsPath: string; names: var seq[string]) =
+  ## Lightweight scan of a `.deps.nif` prelude: collect the raw path text of
+  ## every entry inside an `(include ...)` node (idents like `semexprs`, string
+  ## literals like `"system/mmdisp"`, and the leaves of `a/b` path infixes).
+  ## Liberal by design — it also picks up entries under a statically-false
+  ## `(when ...)`; that is harmless for the only caller (`includerSbifs`), whose
+  ## over-collection just costs an extra, result-free bif scan downstream.
+  if not fileExists(depsPath): return
+  var s = nifstreams.open(depsPath)
+  defer: nifstreams.close(s)
+  discard processDirectives(s.r)
+  var depth = 0
+  var includeDepth = 0     # the `depth` at which the current `(include` opened; 0 = not inside one
+  var t = next(s)
+  while t.kind != EofToken:
+    case t.kind
+    of ParLe:
+      inc depth
+      if includeDepth == 0 and pool.tags[t.tagId] == "include":
+        includeDepth = depth
+    of ParRi:
+      if includeDepth != 0 and depth == includeDepth:
+        includeDepth = 0
+      dec depth
+    of Ident, StringLit:
+      if includeDepth != 0:
+        names.add pool.strings[t.litId]
+    else: discard
+    t = next(s)
+
+proc entryStemBase(roots: seq[string]; name: string): (string, string) =
+  ## Resolve include entry `name` to (deps-stem, base-name); ("","") if unfound.
+  for r in roots:
+    let p = r / name.addFileExt("nim")
+    if fileExists(p):
+      return (moduleSuffix(p, []), splitFile(p).name)
+  result = ("", "")
+
+proc includerSbifs*(conf: ConfigRef; targetFile: AbsoluteFile): seq[string] =
+  ## For an include file `targetFile`, return the `.s.bif` paths of every module
+  ## that includes it — directly OR transitively (following the include chain
+  ## `module -> incA -> incB -> targetFile`). `nim track` uses this to avoid
+  ## loading and scanning every module bif: an include file has no bif of its
+  ## own, so its type-checked tokens live in the *including* module's bif. Only
+  ## the small `.deps.nif` preludes are read here, never a `.s.bif`.
+  const depsExt = ".deps.nif"
+  let nc = getNimcacheDir(conf).string
+
+  # Candidate roots for resolving an `(include X)` entry to a real file, so its
+  # module suffix (== its own deps-file stem) can be computed. Include entries
+  # carry any sub-path (`system/mmdisp`), so the file's *directory* roots suffice:
+  # the target's own dir, the project dir, and the search paths cover the
+  # compiler, the stdlib and typical single-tree projects.
+  var roots: seq[string] = @[parentDir(targetFile.string)]
+  if conf.projectPath.string.len > 0: roots.add conf.projectPath.string
+  for sp in conf.searchPaths: roots.add sp.string
+
+  # One pass over every prelude builds the reverse include graph, keyed by base
+  # file name: `includedBy[b]` = deps stems whose owner directly `include`s a
+  # file named `b`. `stemBase` maps an include-only file's deps stem back to its
+  # own base name, so the walk can climb through nested includes.
+  var includedBy = initTable[string, seq[string]]()
+  var stemBase = initTable[string, string]()
+  for depsPath in walkFiles(nc / "*" & depsExt):
+    let base = extractFilename(depsPath)
+    if base.endsWith(".p" & depsExt): continue   # `.p.deps.nif` twin
+    let ownerStem = base[0 ..< base.len - depsExt.len]
+    var names: seq[string] = @[]
+    collectIncludeNames(depsPath, names)
+    for n in names:
+      let (childStem, childBase) = entryStemBase(roots, n)
+      if childBase.len == 0: continue
+      includedBy.mgetOrPut(childBase, @[]).add ownerStem
+      stemBase[childStem] = childBase          # this child's stem -> its base name
+
+  # Walk UP from the target: a deps stem that includes the current base name is
+  # either a module (has a `.s.bif` -> collect it) or itself an include file
+  # (recurse via its own base name).
+  result = @[]
+  var seenBase = initHashSet[string]()
+  var work = @[splitFile(targetFile.string).name]
+  while work.len > 0:
+    let b = work.pop()
+    if seenBase.containsOrIncl(b): continue
+    for stem in includedBy.getOrDefault(b):
+      let sbif = nc / stem & ".s.bif"
+      if fileExists(sbif):
+        if sbif notin result: result.add sbif   # module owner
+      else:
+        let ob = stemBase.getOrDefault(stem)     # include-only owner: climb higher
+        if ob.len > 0: work.add ob
+
 proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) =
   ## Process a module: run nifler and read deps
   if not runNifler(c, pair.nimFile):

compiler/icconfig.nim

@@ -260,17 +260,26 @@ proc ensureIcConfig*(conf: ConfigRef) =
   if not fileExists(outPath) or sourcesChanged(outPath):
     createDir(cacheDir)
     # Re-invoke ourselves as the config producer: reuse this process's command
-    # line, dropping the command argument (`ic`) in favour of `icconfig` and the
-    # explicit output path, both BEFORE the project file (anything after the
-    # project is swallowed into `config.arguments` by `cmdLineRest`). The
-    # producer re-reads `nim.cfg` itself.
+    # line, dropping the command argument (`ic`/`track`) in favour of `icconfig`
+    # and the explicit output path. Every switch must land BEFORE the project
+    # file, because anything after the project is swallowed into
+    # `config.arguments` by `cmdLineRest` (and a non-empty `arguments` without
+    # `--run` is a hard error). Callers may legitimately put switches after the
+    # project — `nim track PROJ --def:...` — so we re-order rather than replay
+    # verbatim: all `-`-prefixed switches first (in encounter order), then the
+    # non-switch project token(s). The producer re-reads `nim.cfg` itself.
     var pargs = @["icconfig", "--icConfigOut:" & outPath]
+    var rest: seq[string] = @[]
     var droppedCmd = false
     for a in commandLineParams():
-      if not droppedCmd and a.len > 0 and a[0] != '-':
-        droppedCmd = true  # drop the original command token (`ic`)
-      else:
+      if a.len == 0: continue
+      if a[0] == '-':
         pargs.add a
+      elif not droppedCmd:
+        droppedCmd = true  # drop the original command token (`ic`/`track`)
+      else:
+        rest.add a  # project file (and any further non-switch tokens) go last
+    for a in rest: pargs.add a
     let p = startProcess(getAppFilename(), args = pargs,
                          options = {poStdErrToStdOut})
     let outp = p.outputStream.readAll()

compiler/idetools.nim

@@ -35,8 +35,9 @@ import std / [os, strutils, sets]
 import options, msgs, pathutils
 import lineinfos as astli
 import ast2nif   # toNifFilename
+from deps import includerSbifs   # deps-guided include-file lookup
 import "../dist/nimony/src/lib/nifcore"
-from "../dist/nimony/src/lib" / bif import load, BifModule
+from "../dist/nimony/src/lib" / bif import load, BifModule, containsSym
 
 proc identLen(name: string): int =
   ## Length of the displayed identifier: the run before the first `.` of a
@@ -122,21 +123,37 @@ proc emit(conf: ConfigRef; c: Cursor; section: IdeCmd; name: string;
   else:
     conf.suggestWriteln(formatSuggest(s))
 
-proc scanUses(conf: ConfigRef; m: var BifModule; targetName: string;
+proc tokenSymId(c: Cursor): SymId {.inline.} =
+  ## SymId (in the cursor's own per-file pool) of a `Symbol`/`SymbolDef` token,
+  ## or `SymId(0)` for an inline-encoded one — which is never our search target:
+  ## a mangled name (`ident.disamb.suffix`) is always longer than
+  ## `StrInlineMaxLen`, so every occurrence of the symbol we look for is stored by
+  ## pool id, decoded here with a shift and no string materialization.
+  if isInlineLit(c): SymId(0) else: SymId(combinedPayload(c) shr 1)
+
+template symMatches(c: Cursor): bool =
+  ## True when the token at `c` is the searched symbol. The fast path is a pure
+  ## integer compare against `targetSym` (the symbol's id in THIS module's pool,
+  ## resolved once per file by the caller). `targetSym == 0` means the name is not
+  ## representable as a pool id (a rare <=3-byte local): fall back to a string
+  ## compare, correct for both inline and pooled encodings.
+  (if targetSym != SymId(0): tokenSymId(c) == targetSym else: symName(c) == targetName)
+
+proc scanUses(conf: ConfigRef; m: var BifModule; targetSym: SymId; targetName: string;
               seen: var HashSet[string]) =
   ## `--usages`: report every `Symbol` (use) occurrence with valid line info.
   if m.buf.len == 0: return
   var c = m.buf.beginRead()
   while c.hasMore:
-    if c.kind == Symbol and symName(c) == targetName and rawLineInfo(c).isValid:
+    if c.kind == Symbol and symMatches(c) and rawLineInfo(c).isValid:
       emit(conf, c, ideUse, targetName, seen)
     inc c
   c.endRead()
 
-proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string;
+proc scanDef(conf: ConfigRef; m: var BifModule; targetSym: SymId; targetName: string;
              seen: var HashSet[string]) =
-  ## `--def`: report the declaration of `targetName` if this module owns it (has
-  ## its `SymbolDef`). The `SymbolDef` token itself carries no line info; the
+  ## `--def`: report the declaration of the target symbol if this module owns it
+  ## (has its `SymbolDef`). The `SymbolDef` token itself carries no line info; the
   ## declaration location lives on the *enclosing tag* (e.g. `(sd @file:line:col`,
   ## like `bif.buildIndex`'s `mostRecentTagPos`). When that tag has no line info
   ## either, fall back to the declaration-site `Symbol` occurrence — but only in
@@ -153,7 +170,7 @@ proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string;
       mostRecentTagPos = cursorToPosition(m.buf, c)
       inc c
     of SymbolDef:
-      if symName(c) == targetName:
+      if symMatches(c):
         sawDef = true
         var tc = cursorAt(m.buf, mostRecentTagPos)
         if rawLineInfo(tc).isValid:
@@ -162,7 +179,7 @@ proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string;
         tc.endRead()
       inc c
     of Symbol:
-      if fallbackPos < 0 and symName(c) == targetName and rawLineInfo(c).isValid:
+      if fallbackPos < 0 and symMatches(c) and rawLineInfo(c).isValid:
         fallbackPos = cursorToPosition(m.buf, c)
       inc c
     else:
@@ -173,14 +190,32 @@ proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string;
     emit(conf, fc, ideDef, targetName, seen)
     fc.endRead()
 
-proc scanBuf(conf: ConfigRef; m: var BifModule; section: IdeCmd; targetName: string;
-             seen: var HashSet[string]) =
-  ## Emit hits for `targetName` in `m` per the query kind. `ideDus`
+proc scanBuf(conf: ConfigRef; m: var BifModule; section: IdeCmd;
+             targetSym: SymId; targetName: string; seen: var HashSet[string]) =
+  ## Emit hits for the target symbol in `m` per the query kind. `ideDus`
   ## (`--defusages`) reports both the definition and every usage.
   if section in {ideDef, ideDus}:
-    scanDef(conf, m, targetName, seen)
+    scanDef(conf, m, targetSym, targetName, seen)
   if section in {ideUse, ideDus}:
-    scanUses(conf, m, targetName, seen)
+    scanUses(conf, m, targetSym, targetName, seen)
+
+proc findPos(conf: ConfigRef; m: var BifModule; target: TLineInfo;
+             foundName: var string): bool =
+  ## Scan `m` for the `Symbol`/`SymbolDef` token covering the queried position
+  ## `target` and set `foundName` to its mangled name. Returns true on a hit.
+  if m.buf.len == 0: return false
+  var c = m.buf.beginRead()
+  result = false
+  while c.hasMore:
+    let k = c.kind
+    if k == Symbol or k == SymbolDef:
+      let nm = symName(c)
+      if posMatch(c, conf, target, identLen(nm)):
+        foundName = nm
+        result = true
+        break
+    inc c
+  c.endRead()
 
 proc runIdeQuery*(conf: ConfigRef) =
   ## Entry point: called from `main.nim` after `commandCheck` when a
@@ -191,33 +226,54 @@ proc runIdeQuery*(conf: ConfigRef) =
   let target = conf.m.trackPos
   if target.fileIndex.int32 < 0: return
 
-  # Pass 1: position -> symbol, in the queried module's own .s.bif.
+  # Pass 1: position -> symbol. Try the queried file's own module bif first (the
+  # fast path when the position is inside a real module). An include file has no
+  # module bif of its own — its tokens live in the *including* module's bif with
+  # include-file line info — so when the direct lookup misses, consult the
+  # `.deps.nif` preludes (`includerSbifs`) to load only the module(s) that
+  # include the queried file (directly or transitively), never every bif in the
+  # nimcache. `ownerFile` is the bif that owns the hit.
   let modFile = toNifFilename(conf, target.fileIndex)
-  if not fileExists(modFile): return
-  var qm = load(modFile)
   var foundName = ""
-  block find:
-    if qm.buf.len == 0: break find
-    var c = qm.buf.beginRead()
-    while c.hasMore:
-      let k = c.kind
-      if k == Symbol or k == SymbolDef:
-        let nm = symName(c)
-        if posMatch(c, conf, target, identLen(nm)):
-          foundName = nm
-          break find
-      inc c
-    c.endRead()
+  var ownerFile = ""
+  if fileExists(modFile):
+    var qm = load(modFile)
+    if findPos(conf, qm, target, foundName):
+      ownerFile = modFile
+  if foundName.len == 0:
+    for cand in includerSbifs(conf, toFullPath(conf, target.fileIndex).AbsoluteFile):
+      if cand == modFile: continue
+      var m = load(cand)
+      if findPos(conf, m, target, foundName):
+        ownerFile = cand
+        break
   if foundName.len == 0: return
 
   # Pass 2: emit definition / usages. `seen` spans every module so a location is
   # reported once even when scanned across the whole nimcache.
+  #
+  # Cross-file matching is by SymId, not by decoding every token's name. Two
+  # filters keep it cheap:
+  #   1. `bif.containsSym` — a sym-table-only probe that reads just the small
+  #      trailing pools, NOT the token block or any `BiTable`. A module that never
+  #      references the symbol is rejected here without a full `load` (no pools
+  #      built, no token block mapped) — so a query whose symbol lives in a few
+  #      modules no longer pays to load the whole nimcache.
+  #   2. For a module that does contain it, `bif.load` mints a fresh per-file pool,
+  #      so the name is resolved to THIS file's SymId once via `getKeyId`; the scan
+  #      then compares integer ids per token instead of materializing a string for
+  #      each (see `symMatches`).
   var seen = initHashSet[string]()
   if isGlobalName(foundName):
     for f in walkFiles((getNimcacheDir(conf).string) / "*.s.bif"):
+      if not containsSym(f, foundName): continue
       var m = load(f)
-      scanBuf(conf, m, section, foundName, seen)
+      let tid = m.buf.pool.syms.getKeyId(foundName)
+      if tid != SymId(0):
+        scanBuf(conf, m, section, tid, foundName, seen)
   else:
     # Local symbol: its mangled name is not unique across modules, so restrict
-    # the scan to the module it lives in (the queried module).
-    scanBuf(conf, qm, section, foundName, seen)
+    # the scan to the module it lives in (the one that owns the queried position).
+    var qm = load(ownerFile)
+    let tid = qm.buf.pool.syms.getKeyId(foundName)
+    scanBuf(conf, qm, section, tid, foundName, seen)

compiler/options.nim

@@ -29,7 +29,7 @@ const
 
   nimEnableCovariance* = defined(nimEnableCovariance)
 
-  icFormatVersion* = "29"
+  icFormatVersion* = "30"
     ## Version of the IC cache format (the sem-NIF module layout written by
     ## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
     ## that layout changes: `commandIc` wipes a nimcache whose `ic.version`

koch.nim

@@ -16,11 +16,11 @@ const
   ChecksumsStableCommit = "5c132cd332cce5d64a0da9ac3e4c9664313dccb4" # 0.2.2
   SatStableCommit = "9d52513b3c68bfb929dbd687d4fb2836cfee6936"
 
-  NimonyStableCommit = "6f9ac6655dc6724ae4e5ccb93b8123c18d54391a" # unversioned \
+  NimonyStableCommit = "863a04c07277eaa0a6bb2a0059d731a59cbc09b4" # unversioned \
     # Note that Nimony uses Nim as a git submodule but we don't want to install
     # Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive
     # is **required** here.
-    # Commit from 2026-07-03 -- .bif files are memory mapped too
+    # Commit from 2026-07-09 -- .bif files are memory mapped too
 
   # examples of possible values for fusion: #head, #ea82b54, 1.2.3
   FusionStableHash = "#562467452b32cb7a97410ea177f083e6d8405734"