nim track: oneshot nimsuggest (#25971)

This commit is contained in:
Andreas Rumpf
2026-07-08 15:39:40 +02:00
committed by GitHub
parent abdf1ca559
commit f5cf44d7d5
12 changed files with 303 additions and 20 deletions

View File

@@ -118,13 +118,24 @@ proc toClassSymId*(config: ConfigRef; typeId: ItemId): nifstreams.SymId =
type
LineInfoWriter = object
fileK: FileIndex # remember the current pair, even faster than the hash table
# `fileK`/`fileV` cache the most recently resolved (FileIndex -> FileId) pair,
# faster than the hash table. `fileK` MUST be constructed at an invalid
# sentinel (see `newLineInfoWriter`), never zero: `FileIndex(0)` is a real file
# index, and `fileV` zero-inits to `FileId(0)` == `NoFile`, so a zero `fileK`
# would make the first lookup of the module-at-index-0 falsely hit this cache
# and return `NoFile` — silently dropping ALL of that module's line info.
fileK: FileIndex
fileV: FileId
tab: Table[FileIndex, FileId]
revTab: Table[FileId, FileIndex] # reverse mapping for oldLineInfo
man: LineInfoManager
config: ConfigRef
proc newLineInfoWriter(config: ConfigRef): LineInfoWriter =
# `fileK` starts invalid so the one-entry cache never collides with a real
# `FileIndex(0)` (see the type's doc comment).
LineInfoWriter(config: config, fileK: astli.InvalidFileIdx)
proc get(w: var LineInfoWriter; key: FileIndex): FileId =
if w.fileK == key:
result = w.fileV
@@ -883,6 +894,7 @@ var reexpModTag = registerTag("reexpmod")
var offerTag = registerTag("offer")
var typeOfferTag = registerTag("toffer")
var modulesrcTag = registerTag("modulesrc")
var expansionTag = registerTag("expansion")
# `(unusedid <int>)` — the module's first FREE itemId after the frontend
# (`.s.bif`) or the lower stage (`.t.bif`). The backend seeds its per-module
# sym/type counters here so freshly-minted backend ids (closure envs, RTTI
@@ -923,6 +935,7 @@ proc registerNifAstTags*() =
offerTag = registerTag("offer")
typeOfferTag = registerTag("toffer")
modulesrcTag = registerTag("modulesrc")
expansionTag = registerTag("expansion")
proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) =
if n == nil:
@@ -1548,8 +1561,9 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
genericParamsCount: int]] = @[];
typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[];
resolvedImportDeps: seq[FileIndex] = @[];
firstUnusedId: int32 = 0) =
var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule)
firstUnusedId: int32 = 0;
expansions: seq[(PSym, TLineInfo)] = @[]) =
var w = Writer(infos: newLineInfoWriter(config), currentModule: thisModule)
w.deps = newIcBuilder(64)
var content = newIcBuilder(300)
@@ -1626,6 +1640,17 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
w.deps.addStrLit toFullPath(config, FileIndex(thisModule))
w.deps.addParRi
# Template/macro expansions leave no trace in the sem'checked AST, so record
# each as `(expansion <symUse @call-site>)`: a `Symbol` use of the expanded
# routine carrying the ORIGINAL call-site line info. The loader skips the tag
# (processTopLevel), but `idetools` scans every `Symbol` token in the buffer,
# so this restores "find usages / goto-def" for templates and macros.
for (sym, info) in expansions:
if sym == nil: continue
w.deps.addParLe expansionTag, NoLineInfo
w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), trLineInfo(w, info)
w.deps.addParRi
# Generic TYPE-instance OFFERS: the `tyGenericInst` types this module created
# (e.g. `HashArray[8192, Gwei]`). Non-IC keeps ONE such instance in the global
# `typeInstCache`, so a structural bound computed at the first instantiation
@@ -1824,7 +1849,7 @@ type
proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext =
## Supposed to be a global variable
result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache)
result = DecodeContext(infos: newLineInfoWriter(config), cache: cache)
var loadStatsInit {.threadvar.}: int # 0=unknown 1=on 2=off
var statsCtxPtr {.threadvar.}: ptr DecodeContext
@@ -3251,6 +3276,10 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]
# self-identification record for the standalone include-graph scanner;
# not needed by the loader, just skip past it.
skip cur
elif tagIs(cur, "expansion"):
# template/macro expansion usage record for tooling (`idetools` scans it
# as a `Symbol` use); the loader itself needs nothing from it.
skip cur
elif tagIs(cur, "implementation"):
cont = false
elif LoadFullAst in flags or tagIs(cur, toNifTag(nkLetSection)) or
@@ -3312,7 +3341,7 @@ proc writeLoweredModule*(c: var DecodeContext; config: ConfigRef;
# types/globals/params/locals stay Complete and emit real defs (the `.t.nif` is
# the sole source the cg stage reads — no `.s.nif` fallback for them).
sealLoadedRoutines(c)
var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule)
var w = Writer(infos: newLineInfoWriter(config), currentModule: thisModule)
w.deps = newIcBuilder(64)
w.inProc = 1
w.lowering = true

View File

@@ -509,6 +509,7 @@ proc parseCommand*(command: string): Command =
of "nifc": cmdNifC # generate C from NIF files
of "ic": cmdIc # generate .build.nif for nifmake
of "icconfig": cmdIcConfig # produce the precompiled config artifact
of "track": cmdTrack # IDE goto-def / find-usages over `nim ic`'s NIF output
else: cmdUnknown
proc setCmd*(conf: ConfigRef, cmd: Command) =

View File

@@ -1134,8 +1134,12 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.endTree() # stmts
proc commandIc*(conf: ConfigRef) =
## Main entry point for `nim ic`
proc commandIc*(conf: ConfigRef; frontendOnly = false) =
## Main entry point for `nim ic`. With `frontendOnly` (used by `nim track` for
## IDE queries) it runs only Phase 1 — the incremental nifler + `nim m`
## frontend that writes every module's `.s.bif` — and skips the whole-program
## backend (`nim nifc` -> C -> link), which a goto-def / find-usages scan does
## not need.
when not defined(nimKochBootstrap):
let nifler = findNifler()
if nifler.len == 0:
@@ -1275,10 +1279,12 @@ proc commandIc*(conf: ConfigRef) =
if nifmake.len == 0:
rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & buildFile)
# without nifmake we can only print the manual commands; emit the
# backend's too (best effort — discovery cannot run) and stop.
let backendFile = generateBackendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & backendFile)
rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & backendFile)
# backend's too (best effort — discovery cannot run) and stop. An IDE
# query (`frontendOnly`) needs no backend, so skip it there.
if not frontendOnly:
let backendFile = generateBackendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & backendFile)
rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & backendFile)
return
let cmd = quoteShell(nifmake) & " run" & parallel & " " & quoteShell(buildFile)
rawMessage(conf, hintExecuting, cmd)
@@ -1322,7 +1328,9 @@ proc commandIc*(conf: ConfigRef) =
# Phase 2 — backend (whole-program `nim nifc`), run once over the now-final
# graph. Kept a separate nifmake run so backend rebuilds are decided purely
# by nifmake's input mtimes, independent of frontend discovery.
if frontendOk:
# An IDE query (`frontendOnly`) stops after Phase 1: the `.s.bif` it scans
# are all produced by the frontend; codegen + link would be wasted work.
if frontendOk and not frontendOnly:
let backendFile = generateBackendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & backendFile)
let cmd = quoteShell(nifmake) & " run" & parallel & " " & quoteShell(backendFile)

211
compiler/idetools.nim Normal file
View File

@@ -0,0 +1,211 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## NIF-based goto-definition / find-all-usages for `nim track`.
##
## This is the mainline-Nim port of nimony's `idetools.nim`. It answers a
## `--def:FILE,LINE,COL` / `--usages:FILE,LINE,COL` query by *scanning the
## `.s.bif` files* (binary NIF, see `dist/nimony/src/lib/bif.nim`) that the
## preceding `nim ic` frontend (`nim track`) emitted into the nimcache directory
## — NOT by re-running sem. NIF distinguishes a definition (`SymbolDef` token) from a use
## (`Symbol` token) syntactically, so goto-def / find-uses become plain token
## scans over type-checked NIF, which is more reliable than the classic PSym
## engine because generics and macros are type-checked in the NIF too.
##
## Two passes (mirroring nimony's `usages`):
## 1. Load the queried module's `.s.bif` and find the `Symbol`/`SymbolDef`
## token whose line info + identifier length contains `conf.m.trackPos`.
## That yields the mangled symbol NAME and whether it is global (>= 2 dots).
## 2. `--usages`: emit every `Symbol` (use) token; `--def`: every `SymbolDef`.
## A global symbol is scanned across every module `.s.bif`; a local one only
## within the queried module.
##
## IMPORTANT porting note: `bif.load` mints FRESH per-file pools, so a `SymId`
## from module A's buffer is meaningless in module B's. The cross-module match is
## therefore by the mangled NAME string, never by `SymId` (nimony can compare ids
## because it parses every text NIF into one shared global pool; we cannot).
import std / [os, strutils]
import options, msgs, pathutils
import lineinfos as astli
import ast2nif # toNifFilename
import "../dist/nimony/src/lib/nifcore"
from "../dist/nimony/src/lib" / bif import load, BifModule
proc identLen(name: string): int =
## Length of the displayed identifier: the run before the first `.` of a
## mangled NIF name (`ident.disamb[.moduleSuffix]`). Bounds the column match.
let d = name.find('.')
result = if d < 0: name.len else: d
proc isGlobalName(name: string): bool =
## A global symbol carries `ident.disamb.moduleSuffix` (>= 2 dots); a local at
## most `ident.disamb` (<= 1 dot). `moduleSuffix` is a dot-free hash, so a raw
## dot count is equivalent to nifbuilder's suffix-compressed test for our use.
var dots = 0
for i in 1 ..< name.len:
if name[i] == '.': inc dots
result = dots >= 2
proc posMatch(c: Cursor; conf: ConfigRef; target: TLineInfo; tokenLen: int): bool =
## True when `target` (the queried position) falls within the identifier span
## of the Symbol/SymbolDef token at `c`. Mirrors nimony's `lineInfoMatch`; the
## filename is resolved through the loaded buffer's own pool (fresh per file),
## then mapped to a `FileIndex` exactly like `ast2nif.oldLineInfo`.
let li = rawLineInfo(c)
if not li.isValid: return false
if li.line.int != target.line.int: return false
let f = fileInfoIdx(conf, AbsoluteFile lineInfoFile(c))
if f != target.fileIndex: return false
if target.col.int < li.col.int: return false
if target.col.int > li.col.int + tokenLen: return false
result = true
const sep = '\t'
proc formatSuggest(s: Suggest): string =
## Reproduce `suggest.$Suggest` for the `ideDef`/`ideUse` sections without
## importing `suggest` (which would create an import cycle). Layout:
## `section⭾symkind⭾qualifiedPath⭾forth⭾filePath⭾line⭾column⭾⭾quality`.
## symkind is always `skUnknown` here — the raw NIF scan has no PSym to give a
## real kind (like nimony's `foundSymbol`, which leaves it empty).
result = $s.section
result.add sep
result.add "skUnknown"
result.add sep
if s.qualifiedPath.len != 0:
result.add s.qualifiedPath.join(".")
result.add sep
result.add s.forth
result.add sep
result.add s.filePath
result.add sep
result.add $s.line
result.add sep
result.add $s.column
result.add sep # empty doc field (docgen is off outside nimsuggest)
if s.version == 0 or s.version == 3:
result.add sep
result.add $s.quality
proc emit(conf: ConfigRef; c: Cursor; section: IdeCmd; name: string) =
## Report one hit as a nimsuggest-compatible result (routed through the
## structured-output hook / `--stdout`). We only have the mangled name + line
## info from the raw NIF, so symkind/type are left empty — like nimony's
## `foundSymbol`.
let li = rawLineInfo(c)
if not li.isValid: return
let s = Suggest(section: section,
qualifiedPath: @[name[0 ..< identLen(name)]],
filePath: lineInfoFile(c),
line: li.line.int,
column: li.col.int,
tokenLen: identLen(name),
forth: "",
symkind: 0'u8,
quality: 100,
version: conf.suggestVersion)
if conf.suggestionResultHook != nil:
conf.suggestionResultHook(s)
else:
conf.suggestWriteln(formatSuggest(s))
proc scanUses(conf: ConfigRef; m: var BifModule; targetName: string) =
## `--usages`: report every `Symbol` (use) occurrence with valid line info.
if m.buf.len == 0: return
var c = m.buf.beginRead()
while c.hasMore:
if c.kind == Symbol and symName(c) == targetName and rawLineInfo(c).isValid:
emit(conf, c, ideUse, targetName)
inc c
c.endRead()
proc scanDef(conf: ConfigRef; m: var BifModule; targetName: string) =
## `--def`: report the declaration of `targetName` if this module owns it (has
## its `SymbolDef`). The `SymbolDef` token itself carries no line info; the
## declaration location lives on the *enclosing tag* (e.g. `(sd @file:line:col`,
## like `bif.buildIndex`'s `mostRecentTagPos`). When that tag has no line info
## either, fall back to the declaration-site `Symbol` occurrence — but only in
## the owning module, so a plain user of the symbol is never reported as a def.
if m.buf.len == 0: return
var c = m.buf.beginRead()
var mostRecentTagPos = 0
var sawDef = false
var emitted = false
var fallbackPos = -1
while c.hasMore:
case c.kind
of TagLit:
mostRecentTagPos = cursorToPosition(m.buf, c)
inc c
of SymbolDef:
if symName(c) == targetName:
sawDef = true
var tc = cursorAt(m.buf, mostRecentTagPos)
if rawLineInfo(tc).isValid:
emit(conf, tc, ideDef, targetName)
emitted = true
tc.endRead()
inc c
of Symbol:
if fallbackPos < 0 and symName(c) == targetName and rawLineInfo(c).isValid:
fallbackPos = cursorToPosition(m.buf, c)
inc c
else:
inc c
c.endRead()
if sawDef and not emitted and fallbackPos >= 0:
var fc = cursorAt(m.buf, fallbackPos)
emit(conf, fc, ideDef, targetName)
fc.endRead()
proc scanBuf(conf: ConfigRef; m: var BifModule; section: IdeCmd; targetName: string) =
## Emit hits for `targetName` in `m` per the query kind.
if section == ideDef:
scanDef(conf, m, targetName)
else:
scanUses(conf, m, targetName)
proc runIdeQuery*(conf: ConfigRef) =
## Entry point: called from `main.nim` after `commandCheck` when a
## `--def`/`--usages` query is active. Assumes the check just emitted the
## project's `.s.bif` files into `getNimcacheDir(conf)`.
let section = conf.ideCmd
if section notin {ideDef, ideUse}: return
let target = conf.m.trackPos
if target.fileIndex.int32 < 0: return
# Pass 1: position -> symbol, in the queried module's own .s.bif.
let modFile = toNifFilename(conf, target.fileIndex)
if not fileExists(modFile): return
var qm = load(modFile)
var foundName = ""
block find:
if qm.buf.len == 0: break find
var c = qm.buf.beginRead()
while c.hasMore:
let k = c.kind
if k == Symbol or k == SymbolDef:
let nm = symName(c)
if posMatch(c, conf, target, identLen(nm)):
foundName = nm
break find
inc c
c.endRead()
if foundName.len == 0: return
# Pass 2: emit definition / usages.
if isGlobalName(foundName):
for f in walkFiles((getNimcacheDir(conf).string) / "*.s.bif"):
var m = load(f)
scanBuf(conf, m, section, foundName)
else:
# Local symbol: its mangled name is not unique across modules, so restrict
# the scan to the module it lives in (the queried module).
scanBuf(conf, qm, section, foundName)

View File

@@ -34,6 +34,7 @@ from icconfig import produceIcConfig
when not defined(nimKochBootstrap):
import nifbackend
import deps
import idetools
when not defined(leanCompiler):
import docgen
@@ -416,6 +417,20 @@ proc mainCommand*(graph: ModuleGraph) =
for it in conf.searchPaths: msgWriteln(conf, it.string)
of cmdCheck:
commandCheck(graph)
of cmdTrack:
# `nim track --def:/--usages:/--track:` — IDE goto-definition / find-usages.
# Runs `nim ic`'s incremental frontend (nifler + per-module `nim m`, so only
# changed modules recompile and each writes a faithful, VM-executed `.s.bif`
# — covering stdlib too), then scans those NIF files (idetools.runIdeQuery).
# Shares the `nim ic` nimcache dir, so a prior `nim ic` build is reused.
setUseIc(true)
wantMainModule(conf)
setOutFile(conf)
when not defined(nimKochBootstrap):
commandIc(conf, frontendOnly = true)
runIdeQuery(conf)
else:
rawMessage(conf, errGenerated, "nim track not available in bootstrap build")
of cmdM:
# cmdM uses NIF files, not ROD files
graph.config.symbolFiles = disabledSf

View File

@@ -177,6 +177,11 @@ type
procGlobals*: seq[PNode]
nifReplayActions*: Table[int32, seq[PNode]] # module position -> replay actions for NIF
nifExpansions*: Table[int32, seq[(PSym, TLineInfo)]]
# module position -> (template/macro sym, call-site info) for every expansion
# in that module. Templates/macros leave no trace in the sem'checked AST, so
# this side-channel (written into the `.bif`, see ast2nif) is what lets
# `nim track --usages`/`--def` find them. Populated by `rememberExpansion`.
cachedMods: IntSet
hookClosure: IntSet # modules whose serialized hooks were already registered

View File

@@ -120,7 +120,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
# so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the
# driver runs on the exact same config its children will. See icconfig.nim.
when not defined(nimKochBootstrap):
if conf.cmd == cmdIc:
if conf.cmd in {cmdIc, cmdTrack}:
ensureIcConfig(conf)
var graph = newModuleGraph(cache, conf)
@@ -134,7 +134,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
if conf.selectedGC == gcUnselected:
if conf.backend in {backendC, backendCpp, backendObjc} or
(conf.cmd in cmdDocLike and conf.backend != backendJs) or
conf.cmd in {cmdGendepend, cmdNifC, cmdIc, cmdM}:
conf.cmd in {cmdGendepend, cmdNifC, cmdIc, cmdM, cmdTrack}:
initOrcDefines(conf)
if conf.selectedStrings == stringSso and

View File

@@ -205,6 +205,7 @@ type
cmdNifC # generate C code from NIF files
cmdIc # generate .build.nif for nifmake
cmdIcConfig # `nim ic`'s precompiled-config producer (writes ic_config.cfg.nif)
cmdTrack # `nim track --def/--usages`: IC frontend build + NIF scan for IDE queries
const
cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,

View File

@@ -317,9 +317,12 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
# the backend seeds its id minting ABOVE this so closure envs / RTTI hooks
# never share a `toId` with a frontend sym/type. See ast2nif `(unusedid)`.
let firstUnusedId = max(idgen.symId, idgen.typeId)
var expansions: seq[(PSym, TLineInfo)] = @[]
discard graph.nifExpansions.take(module.position.int32, expansions)
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps, reexportedModuleSyms(graph, module),
genericOffers, typeOffers, resolvedImportDeps, firstUnusedId)
genericOffers, typeOffers, resolvedImportDeps, firstUnusedId,
expansions)
# The module's REAL direct imports (incl. macro-generated) for `nim ic`'s
# graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep.
var semDepPaths: seq[string] = @[]

View File

@@ -576,10 +576,12 @@ const
proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
flags: TExprFlags = {}; expectedType: PType = nil): PNode =
rememberExpansion(c, nOrig.info, sym)
let info = getCallLineInfo(n)
# the callee identifier's position is the usage site tooling expects (matches
# `markUsed` below), not the whole-call `nOrig.info`.
rememberExpansion(c, info, sym)
pushInfoContext(c.config, nOrig.info, sym.detailedInfo)
let info = getCallLineInfo(n)
markUsed(c, info, sym)
onUse(info, sym)
if sym == c.p.owner:

View File

@@ -668,7 +668,13 @@ proc rememberExpansion*(c: PContext; info: TLineInfo; expandedSym: PSym) =
## ("find all usages of this template" would not work). We need special
## logic to remember macro/template expansions. This is done here and
## delegated to the "NIF" file mechanism.
discard "XXX To implement"
##
## We only bother when a NIF file is actually going to be written (IC / `nim m`,
## `--compress`, or a running suggestion engine); a plain `nim c` throws the
## record away, so recording it would be pure overhead.
if info.fileIndex == InvalidFileIdx: return
if c.config.cmd == cmdM or optCompress in c.config.globalOptions or c.config.ideActive:
c.graph.nifExpansions.mgetOrPut(c.module.position.int32, @[]).add (expandedSym, info)
const
errVarForOutParamNeededX = "for a 'var' type a variable needs to be passed; but '$1' is immutable"

View File

@@ -26,13 +26,15 @@ const
proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
flags: TExprFlags = {}; expectedType: PType = nil): PNode =
rememberExpansion(c, n.info, s)
let info = getCallLineInfo(n)
# `info` (the callee identifier's position, not the whole call node) is what
# tooling wants to see as the usage site — matches `markUsed` below.
rememberExpansion(c, info, s)
# IC: this expands `s`'s body into the current module's sem, so the module
# depends on that body — record a NeedsImpl (strong) edge to `s`'s module.
# The iface cookie hashes only signatures now, so a template body edit moves
# only the impl cookie, and just the modules that expanded it re-sem.
recordIcImplDep(c.graph, s)
let info = getCallLineInfo(n)
markUsed(c, info, s)
onUse(info, s)
# Note: This is n.info on purpose. It prevents template from creating an info