diff --git a/compiler/ast.nim b/compiler/ast.nim index 068b3c4f86..0d333d91f3 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -332,7 +332,10 @@ when defined(nimsuggest): result = s.allUsagesImpl proc `allUsages=`*(s: PSym, val: sink seq[TLineInfo]) {.inline.} = - assert s.state != Sealed + # No `assert s.state != Sealed`: `allUsagesImpl` is nimsuggest-only usage + # tracking, NOT part of the NIF-serialized symbol. nimsuggest loads symbols + # as `Sealed` (ast2nif.loadedState under cmdM) yet `suggestSym` legitimately + # records usages on them; the getter likewise doesn't assert. if s.state == Partial: loadSym(s) s.allUsagesImpl = val @@ -475,6 +478,8 @@ proc comment*(n: PNode): string = else: result = "" +nodeCommentReader = proc(n: PNode): string {.nimcall.} = comment(n) + proc `comment=`*(n: PNode, a: string) = let id = n.nodeId if a.len > 0: @@ -490,6 +495,8 @@ proc `comment=`*(n: PNode, a: string) = n.flags.excl nfHasComment gconfig.comments.del(id) +nodeCommentWriter = proc(n: PNode; s: string) {.nimcall.} = n.comment = s + # BUGFIX: a module is overloadable so that a proc can have the # same name as an imported module. This is necessary because of # the poor naming choices in the standard library. diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 4323be7cdc..f1f5c787a2 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -11,7 +11,7 @@ import std / [assertions, tables, sets] from std / strutils import startsWith, endsWith, contains -from std / os import fileExists +from std / os import fileExists, dirExists, walkFiles from std / syncio import readFile from std / algorithm import sort import "../dist/checksums/src/checksums" / sha1 @@ -131,6 +131,20 @@ proc nifLineInfo(w: var LineInfoWriter; info: TLineInfo): PackedLineInfo = # Must use pool.man since toString uses pool.man to unpack result = pack(pool.man, fid, info.line.int32, info.col) +proc nifLineInfoWithComment(w: var LineInfoWriter; info: TLineInfo; doc: string): PackedLineInfo = + ## Like `nifLineInfo` but also attaches `doc` as a NIF `#…#` comment on the + ## token. Used to carry `##` doc comments, which the AST serialization itself + ## drops, across a NIF round-trip (the loader reads it back off the info). + if doc.len == 0: + result = nifLineInfo(w, info) + else: + let cid = pool.strings.getOrIncl(doc).uint32 + if info == unknownLineInfo: + result = packWithComment(pool.man, NoFile, 0'i32, 0'i32, cid) + else: + let fid = get(w, info.fileIndex) + result = packWithComment(pool.man, fid, info.line.int32, info.col, cid) + proc oldLineInfo(w: var LineInfoWriter; info: PackedLineInfo): TLineInfo = if info == NoLineInfo: result = unknownLineInfo @@ -185,8 +199,9 @@ type decodedFileIndices: HashSet[FileIndex] locals: HashSet[ItemId] # track proc-local symbols inProc: int - #writtenTypes: seq[PType] # types written in this module, to be unloaded later - #writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later + writtenTypes: seq[PType] # types sealed during this emit; under ideActive + writtenSyms: seq[PSym] # they are reset to Complete afterwards so nimsuggest + # can keep mutating its still-live query targets writtenPackages: HashSet[string] depSuffixes: HashSet[string] # module suffixes already emitted as `(import ...)` deps @@ -361,6 +376,7 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = # module (or nowhere), leaving dangling references (e.g. `symbol has no # offset` for a `pointer` type whose itemId.module drifted away). typ.state = Sealed + if w.infos.config.ideActive: w.writtenTypes.add typ writeTypeDef(w, dest, typ) else: dest.addSymUse pool.syms.getOrIncl(typeToNifSym(typ, w.infos.config)), NoLineInfo @@ -379,8 +395,25 @@ proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) = dest.addStrLit lib.name writeNode w, dest, lib.path +proc docOfSym(sym: PSym): string = + ## The `##` doc comment documenting `sym`, if any (mirrors nifler's + ## docCommentOf). The comment may sit on the decl node itself or as the first + ## `nkCommentStmt` of a routine body. Carried separately on the sym def's NIF + ## token because the AST serialization drops comments — nimsuggest needs it + ## for "find definition" doc hovers. + let n = sym.astImpl + if n == nil or nodeCommentReader == nil: return "" + let own = nodeCommentReader(n) + if own.len > 0: return own + if sym.kindImpl in routineKinds and n.safeLen > bodyPos: + let body = n[bodyPos] + if body != nil and body.kind == nkStmtList and body.len > 0 and + body[0].kind == nkCommentStmt: + return nodeCommentReader(body[0]) + return "" + proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = - dest.addParLe sdefTag, trLineInfo(w, sym.infoImpl) + dest.addParLe sdefTag, nifLineInfoWithComment(w.infos, sym.infoImpl, docOfSym(sym)) dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo # The `x` marker means "importable as a bare identifier into an importer's # scope". Object fields carry `sfExported` (so they are visible via `obj.field` @@ -473,6 +506,7 @@ proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addDotToken() elif shouldWriteSymDef(w, sym): sym.state = Sealed + if w.infos.config.ideActive: w.writtenSyms.add sym writeSymDef(w, dest, sym) else: # NIF has direct support for symbol references so we don't need to use a tag here, @@ -497,6 +531,7 @@ proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = nodeTyp = sym.typImpl if shouldWriteSymDef(w, sym): sym.state = Sealed + if w.infos.config.ideActive: w.writtenSyms.add sym if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): writeType(w, dest, nodeTyp) @@ -609,6 +644,7 @@ var importTag = registerTag("import") var implTag = registerTag("implementation") var reexpModTag = registerTag("reexpmod") var offerTag = registerTag("offer") +var modulesrcTag = registerTag("modulesrc") proc registerNifAstTags*() = ## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag` @@ -640,6 +676,7 @@ proc registerNifAstTags*() = implTag = registerTag("implementation") reexpModTag = registerTag("reexpmod") offerTag = registerTag("offer") + modulesrcTag = registerTag("modulesrc") proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n == nil: @@ -1324,6 +1361,14 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; for ct in off.concreteTypes: w.deps.addSymUse pool.syms.getOrIncl(typeToNifSym(ct, w.infos.config)), NoLineInfo w.deps.addParRi + # Record this module's own absolute source path. The NIF suffix is a hash of + # the (relative) path (gear2/modnames.moduleSuffix) and is NOT reversible, so + # the standalone include-graph scanner (`scanIncludeGraph`, used by nimsuggest + # cold queries) needs the path written explicitly to map an included file back + # to the *source* of its includer without loading the module. + w.deps.addParLe modulesrcTag, NoLineInfo + w.deps.addStrLit toFullPath(config, FileIndex(thisModule)) + w.deps.addParRi # the implTag is used to tell the loader that the # bottom of the file is the implementation of the module: @@ -1350,10 +1395,22 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; let s = op.sym if s.state != Sealed: s.state = Sealed + if config.ideActive: w.writtenSyms.add s writeSymDef w, dest, s dest.addParRi() + # nimsuggest reuses these symbols/types as live, mutable query targets (sem + # re-runs, usage tracking, flag updates). Sealing is only needed for intra-emit + # dedup; once the NIF is built, un-seal so suggest can keep mutating them + # (matches `loadedState` loading Complete under ideActive). The `Sealed` guard + # stays in force for a real `nim m`/`nim nifc` build. + if config.ideActive: + for s in w.writtenSyms: + if s.state == Sealed: s.state = Complete + for t in w.writtenTypes: + if t.state == Sealed: t.state = Complete + # OnlyIfChanged keeps the mtime of content-identical rewrites: nifmake's # mtime-based `needsRebuild` then prunes the rebuild cascade level by # level, and the nifc backend can trust "semmed NIF older than the cnif @@ -1454,15 +1511,29 @@ proc loadedState(c: DecodeContext): ItemState {.inline.} = ## State to give a freshly loaded symbol or type. During the C code generation ## phase (`nim nifc`) the backend (lambda lifting, the transformer, etc.) ## legitimately mutates the loaded entities and never writes them back to a NIF, - ## so they must be mutable (`Complete`). During semantic checking (`nim m`) a - ## loaded entity belongs to an already-compiled dependency and must stay - ## `Sealed` so accidental mutations are caught. - if c.infos.config.cmd == cmdNifC: Complete else: Sealed + ## so they must be mutable (`Complete`). nimsuggest (`ideActive`) is the same + ## case: it reuses loaded symbols as live query targets and mutates them during + ## sem and suggestion bookkeeping (usage tracking, flags) without authoritatively + ## writing those mutations back (its NIF emits are gated to non-dirty, error-free + ## modules and re-serialize from the proper state). During a plain `nim m` + ## semantic check a loaded entity belongs to an already-compiled dependency and + ## must stay `Sealed` so accidental mutations are caught. + if c.infos.config.cmd == cmdNifC or c.infos.config.ideActive: Complete else: Sealed proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry; buf: var TokenBuf): Cursor = let s = addr c.mods[module].stream s.r.jumpTo entry.offset + # A seek-load is self-contained: its tokens must decode their relative line + # info against `entry.info` ALONE. The stream's `parents` stack can be left at + # depth >1 by a prior non-seek read (e.g. loadNifModule reads `(stmts`/ + # `(implementation` without consuming their `)`), and `parse` only overwrites + # parents[0] while `rawNext` reads parents[^1] — so a stale top entry (the last + # symbol decoded, e.g. excpt.nim:746) would become the base. This is benign on + # a freshly-opened stream (cold/stdin) but corrupts EPC recompiles, which reuse + # the per-module stream (the global DecodeContext survives resetAllModules). + # Collapse the stack so parse's parentInfo is the sole base. + s[].parents.setLen 1 nifcursors.parse(s[], buf, entry.info) result = cursorAt(buf, 0) @@ -1693,7 +1764,18 @@ proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string; inc val[] let id = itemId(module.int32, val[]) - let offs = c.getOffset(module, symAsStr) + let offs = c.mods[module].index.getOrDefault(symAsStr) + if offs.offset == 0: + # Only module/package self-syms are never written as `(sd)` entries, so a + # missing index offset means this is such a sym — typically the OWNER of an + # `include`d symbol (`.0.`). Synthesize a resolvable + # skModule stub (itemId item-0 = the module self-sym) instead of asserting + # "symbol has no offset". `Complete` so accessors never try to lazy-load it. + result = PSym(itemId: itemId(module.int32, 0'i32), kindImpl: skModule, + name: c.cache.getIdent(sn.name), disamb: sn.count.int32, + infoImpl: newLineInfo(module, 1, 1), state: Complete) + c.syms[symAsStr] = (result, NifIndexEntry()) + return result let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, offs) @@ -1909,8 +1991,14 @@ proc loadSym*(c: var DecodeContext; s: PSym) = # Now parse the symbol definition with all local symbols pre-registered s.infoImpl = c.infos.oldLineInfo(n.info) + # The `##` doc comment (if any) rides as a NIF comment on the sym def token; + # capture it before advancing, then restore it onto the loaded AST so that + # suggest's `extractDocComment` (findDocComment on `s.ast`) finds it. + let docId = unpack(pool.man, n.info).comment inc n loadSymFromCursor(c, s, n, c.mods[symsModule].suffix, localSyms) + if docId != 0'u32 and s.astImpl != nil and nodeCommentWriter != nil: + nodeCommentWriter(s.astImpl, pool.strings[StrId(docId)]) template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = @@ -2268,6 +2356,10 @@ type ## generic instances this module created; modulegraphs.nim rebuilds ## `procInstCache` from them so a consumer reuses the instance instead of ## re-instantiating it in its own (operator-blind) module scope. + includes*: seq[string] # resolved full paths of files this module `include`s; + # replayed into `inclToMod` by modulegraphs.nim so that + # nimsuggest can map a query in an include file back to + # this module (`parentModule`) and recompile it. proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) = tok = next(s) # skip `(import` @@ -2285,6 +2377,94 @@ proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix] else: raiseAssert "expected ParRi but got " & $tok.kind +proc loadInclude(c: var DecodeContext; s: var Stream; includes: var seq[string]; tok: var PackedToken) = + ## Reads an `(include . . "path"...)` entry written by `trInclude`. The paths + ## are resolved full paths (see semstmts.evalInclude under cmdM/optCompress). + tok = next(s) # skip `(include` + if tok.kind == DotToken: tok = next(s) # flags + if tok.kind == DotToken: tok = next(s) # type + while tok.kind == StringLit: + includes.add pool.strings[tok.litId] + tok = next(s) + if tok.kind == ParRi: + tok = next(s) + else: + raiseAssert "expected ParRi in (include ...) but got " & $tok.kind + +proc scanIncludeGraph*(config: ConfigRef): seq[tuple[includer: string; includes: seq[string]]] = + ## Standalone "full table" scan of every `.nif` in the nimcache: reads + ## only each module's header records — `(modulesrc "path")` (the includer's own + ## source) and `(include . . "path"...)` (resolved included files) — and returns + ## (includerSource, includedSources) pairs for the modules that `include` + ## anything. No `DecodeContext`, no symbol/index loading: it parses the few dep + ## tokens at the top of the file and stops at the first non-dep node. + ## + ## Used by nimsuggest to answer, for a cold-opened *include* file, "which module + ## includes me?" without NIF-loading that module — so the includer can be + ## *source*-compiled (modules that `include` files are never served from NIF). + result = @[] + let dir = getNimcacheDir(config) + if not dirExists(dir.string): return + for f in walkFiles((dir / RelativeFile"*.nif").string): + # only the primary module NIFs; skip the sidecars + # (.iface.nif/.impl.nif/.edges.nif/.s.deps.nif). + if f.endsWith(".iface.nif") or f.endsWith(".impl.nif") or + f.endsWith(".edges.nif") or f.endsWith(".deps.nif"): + continue + var s = nifstreams.open(f) + var includer = "" + var includes: seq[string] = @[] + var t = next(s) # (stmts + if t.kind == ParLe: + t = next(s) # flags dot + t = next(s) # type dot + t = next(s) # first child (matches loadNifModule's priming) + # the dep records (import/include/reexpmod/modulesrc) are written first and + # contiguously; stop at the first body node or the (implementation) marker. + while t.kind == ParLe: + if t.tagId == includeTag or t.tagId == modulesrcTag: + let isInc = t.tagId == includeTag + t = next(s) # into the node (past its ParLe) + while t.kind != ParRi and t.kind != EofToken: + if t.kind == StringLit: + if isInc: includes.add pool.strings[t.litId] + else: includer = pool.strings[t.litId] + t = next(s) + if t.kind == ParRi: t = next(s) # past the ParRi + elif t.tagId == importTag or t.tagId == reexpModTag: + t = skip(s, t) + else: + break + close s + if includer.len > 0 and includes.len > 0: + result.add (includer, includes) + +proc nifModuleHasIncludes*(config: ConfigRef; fileIdx: FileIndex): bool = + ## Cheap header-only check: does the module's `.nif` contain an + ## `(include ...)` record? Used by nimsuggest (`moduleFromNifFile`) to refuse to + ## NIF-serve modules that `include` files, so the includer is source-compiled + ## and the included symbols never round-trip through NIF (which mishandles their + ## owner/line-info on reload). + let f = toNifFilename(config, fileIdx) + if not fileExists(f): return false + var s = nifstreams.open(f) + result = false + var t = next(s) # (stmts + if t.kind == ParLe: + t = next(s) # flags dot + t = next(s) # type dot + t = next(s) # first child + while t.kind == ParLe: + if t.tagId == includeTag: + result = true + break + elif t.tagId == modulesrcTag or t.tagId == importTag or + t.tagId == reexpModTag: + t = skip(s, t) + else: + break + close s + proc addReexportedEnumFields(c: var DecodeContext; sym: PSym; interf: var TStrTable) = ## When a non-pure enum type is (re-)exported, its fields must also become ## visible (unqualified) to importers. In a from-source build this happens via @@ -2396,7 +2576,7 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; ") in export list of module " & suffix & ", last symbol: " & lastGood t = next(s) elif t.tagId == includeTag: - t = skipTree(s) + loadInclude(c, s, result.includes, t) elif t.tagId == importTag: loadImport(c, s, result.deps, t) elif t.tagId == reexpModTag: @@ -2445,6 +2625,10 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; t = next(s) if ok and genSym != nil and instSym != nil: result.genericOffers.add (genSym, instSym, cts, paramsCount) + elif t.tagId == modulesrcTag: + # self-identification record for the standalone include-graph scanner; + # not needed by the lazy loader, just skip past it. + t = skip(s, t) elif t.tagId == implTag: cont = false elif LoadFullAst in flags or t.tagId == letTag or t.tagId == varTag: diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 67f7b02fa5..334bbdc945 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -1150,3 +1150,11 @@ proc strTableGet*(t: TStrTable, name: PIdent): PSym = if result == nil: break if result.name.id == name.id: break h = nextTry(h, high(t.data)) + +# --- doc-comment bridge for the NIF serializer ------------------------------- +# `ast2nif` (the NIF reader/writer) cannot import `ast` (where the comment +# accessor and its `gconfig.comments` side table live) because `ast` imports +# `ast2nif`. These hooks are assigned by `ast` and let the serializer carry a +# decl's `##` doc comment across a NIF round-trip. +var nodeCommentReader*: proc(n: PNode): string {.nimcall.} +var nodeCommentWriter*: proc(n: PNode; s: string) {.nimcall.} diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim index e512486395..5e98071f62 100644 --- a/compiler/cmdlinehelper.nim +++ b/compiler/cmdlinehelper.nim @@ -53,7 +53,8 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) = proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: ConfigRef; graph: ModuleGraph): bool = if self.suggestMode: - conf.setCmd cmdIdeTools + conf.setCmd cmdCheck + conf.ideActive = true if conf.cmd == cmdNimscript: incl(conf.globalOptions, optWasNimscript) loadConfigs(DefaultConfig, cache, conf, graph.idgen) # load all config files diff --git a/compiler/commands.nim b/compiler/commands.nim index 31a185d838..d7dbd3e8b8 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -718,6 +718,14 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.outDir = processPath(conf, arg, info, notRelativeToProj=true) of "usenimcache": processOnOffSwitchG(conf, {optUseNimcache}, arg, pass, info) + of "ideimports": + # nimsuggest: where the import closure comes from. IC is opt-in. + # nif|on load unchanged imports from precompiled NIF (cmdM) + # source|off (default) recompile the whole closure from source (cmdCheck) + case arg.normalize + of "nif", "on", "": conf.ideImportsFromNif = true + of "source", "off": conf.ideImportsFromNif = false + else: localError(conf, info, "'--ideImports' expects 'nif' or 'source', got: '$1'" % arg) of "docseesrcurl": expectArg(conf, switch, arg, pass, info) conf.docSeeSrcUrl = arg diff --git a/compiler/main.nim b/compiler/main.nim index 9052287a1f..fbc2b81be6 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -458,7 +458,7 @@ proc mainCommand*(graph: ModuleGraph) = of cmdJsonscript: setOutFile(graph.config) commandJsonScript(graph) - of cmdUnknown, cmdNone, cmdIdeTools: + of cmdUnknown, cmdNone: rawMessage(conf, errGenerated, "invalid command: " & conf.command) if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop, cmdM} and diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 323ff8fafd..00a7890385 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -607,10 +607,14 @@ proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = when not defined(nimKochBootstrap): # Try to resolve from NIF for both cmdNifC and cmdM (which uses NIF files) if g.config.cmd in {cmdNifC, cmdM}: - # First try system module (most compilerprocs are there) + # First try system module (most compilerprocs are there). + # Only consult the NIF if it actually exists: under nimsuggest's cold + # cache (ideActive) system is compiled from source and has no NIF yet, + # in which case the proc is already registered in-memory and the caller + # found/falls back to it — so degrade to nil instead of asserting. let systemFileIdx = g.config.m.systemFileIdx - if systemFileIdx != InvalidFileIdx and not g.withinSystem: - # Only try to load from NIF if the file exists (it may not during initial ic build) + if systemFileIdx != InvalidFileIdx and not g.withinSystem and + fileExists(toNifFilename(g.config, systemFileIdx)): result = tryResolveCompilerProc(ast.program, name, systemFileIdx) if result != nil: strTableAdd(g.compilerprocs, result) @@ -622,6 +626,7 @@ proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = let module = g.ifaces[moduleIdx].module if module != nil and module.name.s == "threadpool": let threadpoolFileIdx = module.position.FileIndex + if not fileExists(toNifFilename(g.config, threadpoolFileIdx)): break result = tryResolveCompilerProc(ast.program, name, threadpoolFileIdx) if result != nil: strTableAdd(g.compilerprocs, result) @@ -1000,6 +1005,17 @@ when not defined(nimKochBootstrap): if not fileExists(toNifFilename(g.config, fileIdx)): return PrecompiledModule(module: nil) + # NOTE: direction-(c) experiment (refuse to NIF-serve include-bearing modules + # under ideActive, forcing a source compile) is disabled — it reproduces the + # known sibling-resolution corruption (system.string -> excpt.nim:746). The + # cold-include *discovery* scan (scanIncludeGraph) stays; the round-trip + # fidelity of included symbols is the separate, still-open loader problem. + when false: + if g.config.ideActive and not g.withinSystem and + fileIdx != g.config.m.systemFileIdx and + nifModuleHasIncludes(g.config, fileIdx): + return PrecompiledModule(module: nil) + # Create module symbol let filename = AbsoluteFile toFullPath(g.config, fileIdx) @@ -1021,6 +1037,12 @@ when not defined(nimKochBootstrap): let ms = materializeReexportedModule(g, mname, msuffix) if ms != nil: strTableAdd(g.ifaces[fileIdx.int].interf, ms) + # Re-establish include->module mapping so nimsuggest's `parentModule` can map + # a query in an included file back to this (NIF-loaded) module and recompile + # it, exactly as it does for a from-source module. Without this the include + # relationship is invisible for NIF-served modules. + for incPath in result.includes: + g.addIncludeDep(fileIdx, fileInfoIdx(g.config, AbsoluteFile incPath)) # Rebuild `procInstCache` from this module's generic-instance OFFERS so a # consumer's `genericCacheGet` finds the instance and SKIPS re-running @@ -1063,6 +1085,35 @@ when not defined(nimKochBootstrap): if g.config.cmd == cmdM: loadTransitiveHooks(g, result.deps) + proc isModuleFile(g: ModuleGraph; fileIdx: FileIndex): bool = + let i = fileIdx.int32 + i >= 0 and i < g.ifaces.len and g.ifaces[i].module != nil + + proc registerIncluderFromNif*(g: ModuleGraph; fileIdx: FileIndex): bool = + ## Targeted cold-include discovery for nimsuggest: scan the nimcache NIFs + ## (`scanIncludeGraph`) for a module whose include-set contains *this* file + ## and register only that single include->module edge in `inclToMod`, so a + ## query inside the include file resolves its includer via `parentModule`. + ## + ## Deliberately targeted: registering *every* include relationship (i.e. also + ## `system`'s own `include`s) eagerly assigns FileIndexes and pollutes + ## `inclToMod`, which perturbs the NIF line-info decode of unrelated modules + ## (`system.string` then resolves into `excpt.nim`). Touch nothing but the + ## one edge we need. + let target = toFullPath(g.config, fileIdx) + for (includer, includes) in scanIncludeGraph(g.config): + for incFile in includes: + if cmpPaths(incFile, target) == 0: + g.addIncludeDep(fileInfoIdx(g.config, AbsoluteFile includer), fileIdx) + return true + result = false + + proc needsIncludeScan*(g: ModuleGraph; fileIdx: FileIndex): bool = + ## True when `fileIdx` is neither a known module of its own nor an + ## already-known include file — i.e. a cold-opened file whose includer we + ## must still discover via `registerIncluderFromNif`. + not g.isModuleFile(fileIdx) and not g.inclToMod.hasKey(fileIdx) + proc configComplete*(g: ModuleGraph) = #rememberStartupConfig(g.startupPackedConfig, g.config) discard diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 88c17cb440..8c3ef55423 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -351,7 +351,7 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) = ## This is used for 'nim dump' etc. where we don't have nimsuggest ## support. - #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return + #if conf.ideActive and optCDebug notin gGlobalOptions: return let sep = if msgNoUnitSep notin flags: conf.unitSep else: "" if not isNil(conf.writelnHook) and msgSkipHook notin flags: conf.writelnHook(s & sep) @@ -457,8 +457,8 @@ To create a stacktrace, rerun compilation with './koch temp $1 ', see $2 f proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string, ignoreMsg: bool) = if msg in fatalMsgs: - if conf.cmd == cmdIdeTools: log(s) - if conf.cmd != cmdIdeTools or msg != errFatal: + if conf.ideActive: log(s) + if not conf.ideActive or msg != errFatal: quit(conf, msg) if msg >= errMin and msg <= errMax or (msg in warnMin..hintMax and msg in conf.warningAsErrors and not ignoreMsg): @@ -472,7 +472,7 @@ proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string, raiseRecoverableError(s) else: quit(conf, msg) - elif eh == doAbort and conf.cmd != cmdIdeTools: + elif eh == doAbort and not conf.ideActive: quit(conf, msg) elif eh == doRaise: raiseRecoverableError(s) @@ -503,7 +503,7 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) = info = context.info proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool = - msg >= errGenerated and conf.cmd == cmdIdeTools and optIdeDebug notin conf.globalOptions + msg >= errGenerated and conf.ideActive and optIdeDebug notin conf.globalOptions proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) = conf.m.fileInfos[fileIdx.int32].lines.add line @@ -661,7 +661,7 @@ proc warningDeprecated*(conf: ConfigRef, info: TLineInfo = gCmdLineInfo, msg = " message(conf, info, warnDeprecated, msg) proc internalErrorImpl(conf: ConfigRef; info: TLineInfo, errMsg: string, info2: InstantiationInfo) = - if conf.cmd in {cmdIdeTools, cmdCheck} and conf.structuredErrorHook.isNil: return + if (conf.ideActive or conf.cmd == cmdCheck) and conf.structuredErrorHook.isNil: return writeContext(conf, info) liMessage(conf, info, errInternal, errMsg, doAbort, info2) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index fc5f4e3656..822ee844c9 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -316,7 +316,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: if conf.cmd == cmdNimscript: showHintConf() conf.configFiles.setLen 0 - if conf.cmd notin {cmdIdeTools, cmdCheck, cmdDump}: + if not conf.ideActive and conf.cmd notin {cmdCheck, cmdDump}: if conf.cmd == cmdNimscript: runNimScriptIfExists(conf.projectFull, isMain = true) else: diff --git a/compiler/options.nim b/compiler/options.nim index 825addc610..73fc0a7596 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -183,7 +183,6 @@ type cmdCheck # semantic checking for whole project cmdM # only compile a single cmdParse # parse a single file (for debugging) - cmdIdeTools # ide tools (e.g. nimsuggest) cmdNimscript # evaluate nimscript cmdDoc0 cmdDoc # convert .nim doc comments to HTML @@ -400,6 +399,13 @@ type evalMacroCounter*: int exitcode*: int8 cmd*: Command # raw command parsed as enum + ideActive*: bool # serving IDE tooling (nimsuggest): collect suggestions and + # keep going after errors. Decoupled from `cmd` so the IDE + # server can run under any compilation mode (cmdCheck, cmdM). + ideImportsFromNif*: bool # nimsuggest: load the unchanged import closure from + # precompiled NIF (run under cmdM) instead of recompiling it + # from source (cmdCheck). IC is opt-in: default off (cmdCheck); + # `--ideImports:nif` opts in. cmdInput*: string # input command projectIsCmd*: bool # whether we're compiling from a command input implicitCmd*: bool # whether some flag triggered an implicit `command` @@ -685,6 +691,7 @@ proc newConfigRef*(): ConfigRef = command: "", # the main command (e.g. cc, check, scan, etc) commandArgs: @[], # any arguments after the main command commandLine: "", + ideImportsFromNif: false, # IC opt-in; see `--ideImports` implicitImports: @[], # modules that are to be implicitly imported implicitIncludes: @[], # modules that are to be implicitly included docSeeSrcUrl: "", @@ -786,7 +793,7 @@ template quitOrRaise*(conf: ConfigRef, msg = "") = else: quit(msg) # quits with QuitFailure -proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.cmd in cmdDocLike + {cmdIdeTools} +proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.ideActive or conf.cmd in cmdDocLike proc usesWriteBarrier*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc proc usesSso*(conf: ConfigRef): bool {.inline.} = conf.selectedStrings == stringSso @@ -912,7 +919,8 @@ proc getOsCacheDir(): string = proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir = proc nimcacheSuffix(conf: ConfigRef): string = - if conf.cmd == cmdCheck: "_check" + if conf.ideActive: "_nimsuggest" # dedicated cache, never shared with `nim c` + elif conf.cmd == cmdCheck: "_check" elif isDefined(conf, "release") or isDefined(conf, "danger"): "_r" else: "_d" diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 3701d0cfb2..a71ecddac9 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -246,11 +246,20 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator # (imported modules should be loaded from existing NIF files). Members of the # current strongly-connected import group (`--icGroup`) are the exception: # they are compiled from source here, so each must write its own NIF. - let shouldWriteNif = (optCompress in graph.config.globalOptions) or - (graph.config.cmd == cmdM and - (sfMainModule in module.flags or - (graph.config.icGroup.len > 0 and - toFullPath(graph.config, module.position.FileIndex) in graph.config.icGroup))) + let shouldWriteNif = + if graph.config.ideActive: + # nimsuggest (cmdM): persist NIF for cleanly-compiled, SAVED modules so + # later queries load them instead of recompiling. Never persist the + # actively edited buffer (it may hold unsaved/incomplete code) nor a + # module that failed to compile — that would poison the cache. + graph.config.cmd == cmdM and graph.config.errorCounter == 0 and + graph.config.m.fileInfos[module.position].dirtyFile.isEmpty + else: + (optCompress in graph.config.globalOptions) or + (graph.config.cmd == cmdM and + (sfMainModule in module.flags or + (graph.config.icGroup.len > 0 and + toFullPath(graph.config, module.position.FileIndex) in graph.config.icGroup))) if shouldWriteNif and not graph.config.isDefined("nimscript"): topLevelStmts.add finalNode # Collect replay actions from both pragma computations and VM state diff @@ -380,24 +389,31 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF toFullPath(graph.config, fileIdx) notin graph.config.icGroup): let precomp = moduleFromNifFile(graph, fileIdx) if precomp.module == nil: - let nifPath = toNifFilename(graph.config, fileIdx) - # Macro-generated imports (e.g. chronicles' parseStmt("import - # chronicles/textlines") driven by the chronicles_sinks define) are - # invisible to the static scanner, so this module's NIF was never - # built. The importer already recorded this import via - # addImportFileDep, so flush every module's `.s.deps`: `nim ic` reads - # it, re-derives the graph with the missing node + edge, and reruns - # the frontend. We still error — this process cannot finish sem - # without the import — but the discovery is structured data now, not - # a side-channel file. - for importer, deps in graph.importDeps.pairs: - var paths: seq[string] = @[] - for f in deps: paths.add toFullPath(graph.config, f) - writeSemDeps(graph.config, importer.int32, paths) - globalError(graph.config, unknownLineInfo, - "nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) & - " (expected: " & nifPath & ")") - return nil # Don't fall through to compile from source + if graph.config.ideActive: + # nimsuggest bootstrap: this import has no precompiled NIF yet (cold + # cache, or it was invalidated). Don't error — fall through to the + # source-compile path below; the pass-close emits a fresh NIF so the + # next query loads it instead of recompiling. + discard + else: + let nifPath = toNifFilename(graph.config, fileIdx) + # Macro-generated imports (e.g. chronicles' parseStmt("import + # chronicles/textlines") driven by the chronicles_sinks define) are + # invisible to the static scanner, so this module's NIF was never + # built. The importer already recorded this import via + # addImportFileDep, so flush every module's `.s.deps`: `nim ic` reads + # it, re-derives the graph with the missing node + edge, and reruns + # the frontend. We still error — this process cannot finish sem + # without the import — but the discovery is structured data now, not + # a side-channel file. + for importer, deps in graph.importDeps.pairs: + var paths: seq[string] = @[] + for f in deps: paths.add toFullPath(graph.config, f) + writeSemDeps(graph.config, importer.int32, paths) + globalError(graph.config, unknownLineInfo, + "nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) & + " (expected: " & nifPath & ")") + return nil # Don't fall through to compile from source else: # Module successfully loaded from NIF file - use it and skip processing result = precomp.module @@ -520,13 +536,21 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx graph.config.m.systemFileIdx = fileInfoIdx(graph.config, graph.config.libpath / RelativeFile"system.nim") when not defined(nimKochBootstrap): - let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx) - graph.systemModule = precomp.module + # Don't clobber an already-compiled system: nimsuggest's NimScript config + # evaluation compiles `system` into this same graph before we get here. if graph.systemModule == nil: - let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx) - localError(graph.config, unknownLineInfo, - "nim m requires precompiled NIF for system module (expected: " & nifPath & ")") - return + let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx) + graph.systemModule = precomp.module + if graph.systemModule == nil: + if graph.config.ideActive: + # nimsuggest bootstrap: no system NIF yet — compile it from source + # (the pass-close emits it), then continue with the main module. + graph.compilePipelineSystemModule() + else: + let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx) + localError(graph.config, unknownLineInfo, + "nim m requires precompiled NIF for system module (expected: " & nifPath & ")") + return discard graph.compilePipelineModule(projectFile, {sfMainModule}) else: graph.compilePipelineSystemModule() diff --git a/compiler/sem.nim b/compiler/sem.nim index 54042d3fb2..a9f87cd006 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -77,7 +77,7 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode; # templates perform some quick check whether the cursor is actually in # the generic or template. when defined(nimsuggest): - if c.config.cmd == cmdIdeTools and requiresCheck: + if c.config.ideActive and requiresCheck: #if optIdeDebug in gGlobalOptions: # echo "passing to safeSemExpr: ", renderTree(n) discard safeSemExpr(c, n) @@ -874,7 +874,7 @@ proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = result = hloStmt(c, result) if c.config.cmd == cmdInteractive and not isEmptyType(result.typ): result = buildEchoStmt(c, result) - if c.config.cmd == cmdIdeTools: + if c.config.ideActive: appendToModule(c.module, result) trackStmt(c, c.module, result, isTopLevel = true) if optMultiMethods notin c.config.globalOptions and @@ -911,7 +911,7 @@ proc semWithPContext*(c: PContext, n: PNode): PNode = result = nil else: result = newNodeI(nkEmpty, n.info) - #if c.config.cmd == cmdIdeTools: findSuggest(c, n) + #if c.config.ideActive: findSuggest(c, n) proc reportUnusedModules(c: PContext) = if c.config.cmd == cmdM: return @@ -920,7 +920,7 @@ proc reportUnusedModules(c: PContext) = message(c.config, info, warnUnusedImportX, s.name.s) proc closePContext*(graph: ModuleGraph; c: PContext, n: PNode): PNode = - if c.config.cmd == cmdIdeTools and not c.suggestionsMade: + if c.config.ideActive and not c.suggestionsMade: suggestSentinel(c) closeScope(c) # close module's scope rawCloseScope(c) # imported symbols; don't check for unused ones! diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 3bbbc1b97e..ef3c58afce 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1571,7 +1571,7 @@ proc builtinFieldAccess(c: PContext; n: PNode; flags: var TExprFlags): PNode = # here at all! #if isSymChoice(n[1]): return when defined(nimsuggest): - if c.config.cmd == cmdIdeTools: + if c.config.ideActive: suggestExpr(c, n) if exactEquals(c.config.m.trackPos, n[1].info): suggestExprNoCheck(c, n) @@ -3405,7 +3405,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType c.config.expandNodeResult = $n suggestQuit() - if c.config.cmd == cmdIdeTools: suggestExpr(c, n) + if c.config.ideActive: suggestExpr(c, n) if nfSem in n.flags: return case n.kind of nkIdent, nkAccQuoted: diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 1d12ae970a..91a834078d 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -273,7 +273,7 @@ proc semGenericStmt(c: PContext, n: PNode, when defined(nimsuggest): if withinTypeDesc in flags: inc c.inTypeContext - #if conf.cmd == cmdIdeTools: suggestStmt(c, n) + #if conf.ideActive: suggestStmt(c, n) semIdeForTemplateOrGenericCheck(c.config, n, ctx.cursorInBody) case n.kind diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 94a38bd4f1..04bc2e3ee5 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -531,7 +531,7 @@ proc semUsing(c: PContext; n: PNode): PNode = if not isTopLevel(c): localError(c.config, n.info, errXOnlyAtModuleScope % "using") for i in 0.. 0: let pending = move c.forwardTypeUpdates var madeProgress = false @@ -1829,7 +1829,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) = c.forwardTypeUpdates.add (owner, typ, typeNode) elif not remainingOwners.missingOrExcl(owner.id): madeProgress = true - + if not madeProgress: # can't error here unfortunately break @@ -2868,7 +2868,7 @@ proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult, resolvedIncStmt proc evalInclude(c: PContext, n: PNode): PNode = result = newNodeI(nkStmtList, n.info) var resolvedIncStmt: PNode = nil - if optCompress in c.config.globalOptions: + if optCompress in c.config.globalOptions or c.config.cmd == cmdM: # New resolve the include filenames to string literals that contain absolute paths, # nicer for IC: resolvedIncStmt = newNodeI(nkIncludeStmt, n.info) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 296098575b..a026fc994a 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -2233,7 +2233,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = nil inc c.inTypeContext - if c.config.cmd == cmdIdeTools: suggestExpr(c, n) + if c.config.ideActive: suggestExpr(c, n) case n.kind of nkEmpty: result = n.typ of nkTypeOfExpr: diff --git a/compiler/suggest.nim b/compiler/suggest.nim index a1cd8b9237..bf7a9faf81 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -911,7 +911,7 @@ proc suggestDecl*(c: PContext, n: PNode; s: PSym) = defer: if attached: dec(c.inTypeContext) # If user is typing out an enum field, then don't provide suggestions - if s.kind == skEnumField and c.config.cmd == cmdIdeTools and exactEquals(c.config.m.trackPos, n.info): + if s.kind == skEnumField and c.config.ideActive and exactEquals(c.config.m.trackPos, n.info): suggestQuit() suggestExpr(c, n) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index 650d6a3c1d..0d8921905c 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -35,7 +35,7 @@ import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp # suggestionResultHook, because suggest.nim is included by sigmatch. # So we import that one instead. import compiler / [options, commands, modules, - passes, passaux, msgs, + passes, passaux, msgs, pipelines, sigmatch, ast, idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper, pathutils, condsyms, syntaxes, suggestsymdb] @@ -261,6 +261,14 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, var isKnownFile = true let dirtyIdx = fileInfoIdx(conf, file, isKnownFile) + # Cold-opened include file: nothing has been compiled/loaded yet, so its + # includer is unknown. Scan the nimcache NIFs for the module that `include`s + # this exact file and register that one edge; `parentModule(dirtyIdx)` then + # resolves to the includer, which we (re)compile below. + if conf.ideImportsFromNif and graph.needsIncludeScan(dirtyIdx): + discard graph.registerIncluderFromNif(dirtyIdx) + let isInclude = graph.inclToMod.hasKey(dirtyIdx) + if not dirtyfile.isEmpty: msgs.setDirtyFile(conf, dirtyIdx, dirtyfile) else: msgs.setDirtyFile(conf, dirtyIdx, AbsoluteFile"") @@ -269,9 +277,9 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, conf.errorCounter = 0 if conf.suggestVersion == 1: graph.usageSym = nil - if not isKnownFile: + if not isKnownFile and not isInclude: graph.clearInstCache(dirtyIdx) - graph.compileProject(dirtyIdx) + graph.compilePipelineProject(dirtyIdx) if conf.suggestVersion == 0 and conf.ideCmd in {ideUse, ideDus} and dirtyfile.isEmpty: discard "no need to recompile anything" @@ -279,10 +287,19 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, let modIdx = graph.parentModule(dirtyIdx) graph.markDirty dirtyIdx graph.markClientsDirty dirtyIdx + # For an include-file query the includer must be re-sem'd so the include body + # (where trackPos sits) is re-checked. An already-loaded includer is only + # recompiled when dirty (pipelines.compilePipelineModule), and the include + # edge isn't always in `g.deps` for markClientsDirty to catch (notably on the + # EPC path), so mark the includer dirty explicitly. + if isInclude: + graph.markDirty modIdx if conf.ideCmd != ideMod: - if isKnownFile: + # `isInclude`: a freshly discovered include file is not "known" yet, but we + # still must (source-)compile its includer to serve the query. + if isKnownFile or isInclude: graph.clearInstCache(modIdx) - graph.compileProject(modIdx) + graph.compilePipelineProject(modIdx) if conf.ideCmd in {ideUse, ideDus}: let u = if conf.suggestVersion != 1: graph.symFromInfo(conf.m.trackPos) else: graph.usageSym if u != nil: @@ -577,7 +594,7 @@ proc recompileFullProject(graph: ModuleGraph) = graph.vm = nil graph.resetAllModules() GC_fullCollect() - graph.compileProject() + graph.compilePipelineProject() proc mainThread(graph: ModuleGraph) = let conf = graph.config @@ -621,10 +638,14 @@ var proc mainCommand(graph: ModuleGraph) = let conf = graph.config - clearPasses(graph) - registerPass graph, verbosePass - registerPass graph, semPass - conf.setCmd cmdIdeTools + # Use the pipeline driver (same as `nim check`): it is where IC/NIF loading + # and emission live. The legacy `passes.compileProject` path has no NIF support. + setPipeLinePass(graph, SemPass) + # cmdM loads the unchanged import closure from precompiled NIF; cmdCheck + # recompiles everything from source. `ideActive` keeps suggestion collection + # and error-resilience regardless of which mode we run under. + conf.setCmd(if conf.ideImportsFromNif: cmdM else: cmdCheck) + conf.ideActive = true defineSymbol(conf.symbols, $conf.backend) wantMainModule(conf) @@ -646,7 +667,7 @@ proc mainCommand(graph: ModuleGraph) = # compile the project before showing any input so that we already # can answer questions right away: benchmark "Initial compilation": - compileProject(graph) + compilePipelineProject(graph) open(requests) open(results) @@ -798,7 +819,7 @@ proc recompilePartially(graph: ModuleGraph, projectFileIdx = InvalidFileIdx) = try: benchmark "Recompilation": - graph.compileProject(projectFileIdx) + graph.compilePipelineProject(projectFileIdx) except Exception as e: myLog fmt "Failed to recompile partially with the following error:\n {e.msg} \n\n {e.getStackTrace()}" try: @@ -1078,9 +1099,22 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, myLog fmt "cmd: {cmd}, file: {file}[{line}:{col}], dirtyFile: {dirtyfile}, tag: {tag}" var fileIndex: FileIndex = default(FileIndex) + # The module to (re)compile for this query. For a normal file it is the file + # itself; for an `include` file it is the module that includes it — the include + # body is only sem'd as part of its includer, and the includer compiles as the + # main module (source, never NIF-loaded), so the include statements at the + # cursor get re-checked. The query position stays in the include file. + var moduleToCompile: FileIndex = default(FileIndex) + var isIncludeQuery = false if not (cmd in {ideRecompile, ideGlobalSymbols}): fileIndex = fileInfoIdx(conf, file) + # Discover an include file's includer from the NIF include graph (cold query) + # so `parentModule` can map it; see registerIncluderFromNif. + if conf.ideImportsFromNif and graph.needsIncludeScan(fileIndex): + discard graph.registerIncluderFromNif(fileIndex) + isIncludeQuery = graph.inclToMod.hasKey(fileIndex) + moduleToCompile = if isIncludeQuery: graph.parentModule(fileIndex) else: fileIndex msgs.setDirtyFile( conf, fileIndex, @@ -1100,14 +1134,20 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, # these commands require partially compiled project elif cmd in {ideSug, ideCon, ideOutline, ideHighlight, ideDef, ideChkFile, ideType, ideDeclaration, ideExpand} and - (graph.needsCompilation(fileIndex) or cmd in {ideSug, ideCon}): + (graph.needsCompilation(fileIndex) or cmd in {ideSug, ideCon} or isIncludeQuery): # for ideSug use v2 implementation if cmd in {ideSug, ideCon}: conf.m.trackPos = newLineInfo(fileIndex, line, col) conf.m.trackPosAttached = false else: conf.m.trackPos = default(TLineInfo) - graph.recompilePartially(fileIndex) + # An include file's includer must be (re)compiled from source so the + # include body is re-sem'd; force it dirty since the include file itself + # is not a module the dirty machinery tracks. + if isIncludeQuery: + graph.markDirty moduleToCompile + graph.markClientsDirty moduleToCompile + graph.recompilePartially(moduleToCompile) case cmd of ideDef: @@ -1149,11 +1189,15 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, graph.markDirtyIfNeeded(file.string, fileIndex) of ideSug, ideCon: # ideSug/ideCon performs partial build of the file, thus mark it dirty for the - # future calls. + # future calls. For an include file, drive everything off its includer module + # (the include file has no module of its own — getModule would be nil). graph.markDirtyIfNeeded(file.string, fileIndex) - graph.recompilePartially(fileIndex) - let m = graph.getModule fileIndex - incl m, sfDirty + if isIncludeQuery: + graph.markClientsDirty fileIndex + graph.recompilePartially(moduleToCompile) + let m = graph.getModule moduleToCompile + if m != nil: + incl m, sfDirty of ideOutline: let n = parseFile(fileIndex, graph.cache, graph.config) graph.iterateOutlineNodes(n, graph.fileSymbols(fileIndex).deduplicateSymInfoPair(false)) @@ -1311,11 +1355,10 @@ else: proc mockCommand(graph: ModuleGraph) = retval = graph let conf = graph.config - conf.setCmd cmdIdeTools + conf.setCmd(if conf.ideImportsFromNif: cmdM else: cmdCheck) + conf.ideActive = true defineSymbol(conf.symbols, $conf.backend) - clearPasses(graph) - registerPass graph, verbosePass - registerPass graph, semPass + setPipeLinePass(graph, SemPass) wantMainModule(conf) @@ -1331,7 +1374,7 @@ else: # compile the project before showing any input so that we already # can answer questions right away: - compileProject(graph) + compilePipelineProject(graph) proc mockCmdLine(pass: TCmdLinePass, cmd: string; conf: ConfigRef) = diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index b69569288d..e0ed3ca337 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -22,6 +22,14 @@ const import std/compilesettings +# nimsuggest's incremental (NIF/IC) mode is opt-in via `--ideImports:nif`. By default +# nimsuggest recompiles the import closure from source (cmdCheck), which is fast and +# stable, so the whole suite runs that path. Only the tests listed below exercise the +# IC path; running every test under IC dominated the suite's wall-clock (each test +# recompiles `system` cold into NIF, plus a few warm-cache-only ordering/highlight +# quirks) and blew CI's job timeout. +const icTests = ["tic.nim", "tv3_import.nim"] + proc parseTest(filename: string; epcMode=false): Test = const cursorMarker = "#[!]#" let nimsug = "bin" / addFileExt("nimsuggest_testing", ExeExt) @@ -74,6 +82,14 @@ proc parseTest(filename: string; epcMode=false): Test = # else: ignore empty lines for better readability of the specs inc i tmp.close() + # The IC tests opt into the NIF path and get their own private cache. The stdio + # variant (epcMode=false) starts cold and writes the NIFs; the EPC variant reuses + # them warm, so a single test exercises both the NIF write and the NIF read path. + if extractFilename(filename) in icTests: + let nimcache = getTempDir() / ("nimsuggest_ic_" & + extractFilename(result.dest).changeFileExt("")) + if not epcMode: removeDir(nimcache) + result.cmd.add " --ideImports:nif --nimcache:" & nimcache # now that we know the markers, substitute them: for a in mitems(result.script): a[0] = a[0] % markers diff --git a/nimsuggest/tests/ttemplate_highlight.nim b/nimsuggest/tests/ttemplate_highlight.nim index 2cbac3be5e..ecc2588c97 100644 --- a/nimsuggest/tests/ttemplate_highlight.nim +++ b/nimsuggest/tests/ttemplate_highlight.nim @@ -1,7 +1,7 @@ doAssert true#[!]# discard """ -$nimsuggest --tester $1 +$nimsuggest --tester $file >highlight $1 highlight;;skTemplate;;1;;0;;8 highlight;;skTemplate;;1;;0;;8