Initial implementation of nimsuggest v3 (#19826)

* Initial implementation of nimsuggest v3

Rework `nimsuggest` to use caching to make usage of ide commands more efficient.
Previously, all commands no matter what the state of the process is were causing
clean build. In the context of Language Server Protocol(LSP) and lsp clients
this was causing perf issues and overall instability. Overall, the goal of v3 is
to fit to LSP Server needs

- added two new commands:
  - `recompile` to do clean compilation
  - `changed` which can be used by the IDEs to notify that a particular file has been changed.
The later can be utilized when using LSP file watches.
  - `globalSymbols` - searching global references

- added `segfaults` dependency to allow fallback to clean build when incremental
fails. I wish the error to be propagated to the client so we can work on fixing
the incremental build failures (typically hitting pointer)

- more efficient rebuild flow. ATM incremental rebuild is triggered when the
command needs that(i. e. it is global) while the commands that work on the
current source rebuild only it

Things missing in this PR:

- Documentation
- Extensive unit testing.

Although functional I still see this more as a POC that this approach can work.

Next steps:
- Implement `sug` request.
- Rework/extend the protocol to allow better client/server communication.
Ideally we will need push events, diagnostics should be restructored to allow
per file notifications, etc.
- implement v3 test suite.
- better logging

* Add tests for v3 and implement ideSug

* Remove typeInstCache/procInstCache cleanup

* Add ideChkFile command

* Avoid contains call when adding symbol info

* Remove log

* Remove segfaults

compiler/modulegraphs.nim

@@ -11,7 +11,7 @@
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
 
-import intsets, tables, hashes, md5_old
+import intsets, tables, hashes, md5_old, sequtils
 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages
 import ic / [packed_ast, ic]
 
@@ -83,6 +83,8 @@ type
     doStopCompile*: proc(): bool {.closure.}
     usageSym*: PSym # for nimsuggest
     owners*: seq[PSym]
+    suggestSymbols*: Table[FileIndex, seq[tuple[sym: PSym, info: TLineInfo]]]
+    suggestErrors*: Table[FileIndex, seq[Suggest]]
     methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization!
     systemModule*: PSym
     sysTypes*: array[TTypeKind, PType]
@@ -385,9 +387,19 @@ when defined(nimfind):
       c.graph.onDefinitionResolveForward(c.graph, s, info)
 
 else:
-  template onUse*(info: TLineInfo; s: PSym) = discard
-  template onDef*(info: TLineInfo; s: PSym) = discard
-  template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
+  when defined(nimsuggest):
+    template onUse*(info: TLineInfo; s: PSym) = discard
+
+    template onDef*(info: TLineInfo; s: PSym) =
+      let c = getPContext()
+      if c.graph.config.suggestVersion == 3:
+        suggestSym(c.graph, info, s, c.graph.usageSym)
+
+    template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
+  else:
+    template onUse*(info: TLineInfo; s: PSym) = discard
+    template onDef*(info: TLineInfo; s: PSym) = discard
+    template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
 
 proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
@@ -434,8 +446,7 @@ proc initOperators*(g: ModuleGraph): Operators =
   result.opNot = createMagic(g, "not", mNot)
   result.opContains = createMagic(g, "contains", mInSet)
 
-proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
-  result = ModuleGraph()
+proc initModuleGraphFields(result: ModuleGraph) =
   # A module ID of -1 means that the symbol is not attached to a module at all,
   # but to the module graph:
   result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32)
@@ -445,9 +456,9 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.ifaces = @[]
   result.importStack = @[]
   result.inclToMod = initTable[FileIndex, FileIndex]()
-  result.config = config
-  result.cache = cache
   result.owners = @[]
+  result.suggestSymbols = initTable[FileIndex, seq[tuple[sym: PSym, info: TLineInfo]]]()
+  result.suggestErrors = initTable[FileIndex, seq[Suggest]]()
   result.methods = @[]
   initStrTable(result.compilerprocs)
   initStrTable(result.exposed)
@@ -461,6 +472,12 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.operators = initOperators(result)
   result.emittedTypeInfo = initTable[string, FileIndex]()
 
+proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
+  result = ModuleGraph()
+  result.config = config
+  result.cache = cache
+  initModuleGraphFields(result)
+
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)
   g.deps = initIntSet()
@@ -472,6 +489,7 @@ proc resetAllModules*(g: ModuleGraph) =
   g.methods = @[]
   initStrTable(g.compilerprocs)
   initStrTable(g.exposed)
+  initModuleGraphFields(g)
 
 proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
   if fileIdx.int32 >= 0:
@@ -550,7 +568,19 @@ proc transitiveClosure(g: var IntSet; n: int) =
 
 proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   let m = g.getModule fileIdx
-  if m != nil: incl m.flags, sfDirty
+  if m != nil:
+    g.suggestSymbols.del(fileIdx)
+    g.suggestErrors.del(fileIdx)
+    incl m.flags, sfDirty
+
+proc unmarkAllDirty*(g: ModuleGraph) =
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil:
+      m.flags.excl sfDirty
+
+proc isDirty*(g: ModuleGraph; m: PSym): bool =
+  result = g.suggestMode and sfDirty in m.flags
 
 proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   # we need to mark its dependent modules D as dirty right away because after
@@ -562,12 +592,26 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
 
   # every module that *depends* on this file is also dirty:
   for i in 0i32..<g.ifaces.len.int32:
-    let m = g.ifaces[i].module
-    if m != nil and g.deps.contains(i.dependsOn(fileIdx.int)):
-      incl m.flags, sfDirty
+    if g.deps.contains(i.dependsOn(fileIdx.int)):
+      g.markDirty(FileIndex(i))
 
-proc isDirty*(g: ModuleGraph; m: PSym): bool =
-  result = g.suggestMode and sfDirty in m.flags
+proc needsCompilation*(g: ModuleGraph): bool =
+  # every module that *depends* on this file is also dirty:
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil:
+      if sfDirty in m.flags:
+        return true
+
+proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool =
+  let module = g.getModule(fileIdx)
+  if module != nil and g.isDirty(module):
+    return true
+
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil and g.isDirty(m) and g.deps.contains(fileIdx.int32.dependsOn(i)):
+      return true
 
 proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
   result = s.ast[bodyPos]
@@ -611,3 +655,13 @@ proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
 func belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool =
   ## Check if symbol belongs to the 'stdlib' package.
   sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId
+
+iterator suggestSymbolsIter*(g: ModuleGraph): tuple[sym: PSym, info: TLineInfo] =
+  for xs in g.suggestSymbols.values:
+    for x in xs.deduplicate:
+      yield x
+
+iterator suggestErrorsIter*(g: ModuleGraph): Suggest =
+  for xs in g.suggestErrors.values:
+    for x in xs:
+      yield x

compiler/options.nim

@@ -188,8 +188,9 @@ type
     # as far as usesWriteBarrier() is concerned
 
   IdeCmd* = enum
-    ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod,
-    ideHighlight, ideOutline, ideKnown, ideMsg, ideProject
+    ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod,
+    ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols,
+    ideRecompile, ideChanged
 
   Feature* = enum  ## experimental features; DO NOT RENAME THESE!
     implicitDeref,
@@ -993,12 +994,16 @@ proc parseIdeCmd*(s: string): IdeCmd =
   of "use": ideUse
   of "dus": ideDus
   of "chk": ideChk
+  of "chkFile": ideChkFile
   of "mod": ideMod
   of "highlight": ideHighlight
   of "outline": ideOutline
   of "known": ideKnown
   of "msg": ideMsg
   of "project": ideProject
+  of "globalSymbols": ideGlobalSymbols
+  of "recompile": ideRecompile
+  of "changed": ideChanged
   else: ideNone
 
 proc `$`*(c: IdeCmd): string =
@@ -1009,6 +1014,7 @@ proc `$`*(c: IdeCmd): string =
   of ideUse: "use"
   of ideDus: "dus"
   of ideChk: "chk"
+  of ideChkFile: "chkFile"
   of ideMod: "mod"
   of ideNone: "none"
   of ideHighlight: "highlight"
@@ -1016,6 +1022,9 @@ proc `$`*(c: IdeCmd): string =
   of ideKnown: "known"
   of ideMsg: "msg"
   of ideProject: "project"
+  of ideGlobalSymbols: "globalSymbols"
+  of ideRecompile: "recompile"
+  of ideChanged: "changed"
 
 proc floatInt64Align*(conf: ConfigRef): int16 =
   ## Returns either 4 or 8 depending on reasons.

compiler/passes.nim

@@ -14,7 +14,7 @@ import
   options, ast, llstream, msgs,
   idents,
   syntaxes, modulegraphs, reorder,
-  lineinfos, pathutils, packages
+  lineinfos, pathutils, std/sha1, packages
 
 when defined(nimPreviewSlimSystem):
   import std/syncio
@@ -132,6 +132,11 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
       return false
   else:
     s = stream
+
+  when defined(nimsuggest):
+    let filename = toFullPathConsiderDirty(graph.config, fileIdx).string
+    msgs.setHash(graph.config, fileIdx, $sha1.secureHashFile(filename))
+
   while true:
     openParser(p, fileIdx, s, graph.cache, graph.config)
 

compiler/suggest.nim

@@ -117,7 +117,7 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int
     elif sourceIdent != ident:
       result = 0
 
-proc symToSuggest(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
+proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
                   quality: range[0..100]; prefix: PrefixMatch;
                   inTypeContext: bool; scope: int;
                   useSuppliedInfo = false): Suggest =
@@ -203,14 +203,14 @@ proc `$`*(suggest: Suggest): string =
     result.add(sep)
     when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler):
       result.add(suggest.doc.escape)
-    if suggest.version == 0:
+    if suggest.version in {0, 3}:
       result.add(sep)
       result.add($suggest.quality)
       if suggest.section == ideSug:
         result.add(sep)
         result.add($suggest.prefix)
 
-proc suggestResult(conf: ConfigRef; s: Suggest) =
+proc suggestResult*(conf: ConfigRef; s: Suggest) =
   if not isNil(conf.suggestionResultHook):
     conf.suggestionResultHook(s)
   else:
@@ -424,7 +424,7 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
         t = skipTypes(t[0], skipPtrs)
     elif typ.kind == tyTuple and typ.n != nil:
       suggestSymList(c, typ.n, field, n.info, outputs)
-    
+
     suggestOperations(c, n, field, orig, outputs)
     if typ != orig:
       suggestOperations(c, n, field, typ, outputs)
@@ -482,7 +482,7 @@ proc findDefinition(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym
   if s.isNil: return
   if isTracked(info, g.config.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags):
     suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym))
-    if sfForward notin s.flags:
+    if sfForward notin s.flags and g.config.suggestVersion != 3:
       suggestQuit()
     else:
       usageSym = s
@@ -497,6 +497,8 @@ proc suggestSym*(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym; i
   ## misnamed: should be 'symDeclared'
   let conf = g.config
   when defined(nimsuggest):
+    g.suggestSymbols.mgetOrPut(info.fileIndex, @[]).add (s, info)
+
     if conf.suggestVersion == 0:
       if s.allUsages.len == 0:
         s.allUsages = @[info]

nimsuggest/nimsuggest.nim

@@ -7,6 +7,12 @@
 #    distribution, for details about the copyright.
 #
 
+import compiler/renderer
+import strformat
+import tables
+import std/sha1
+import times
+
 ## Nimsuggest is a tool that helps to give editors IDE like capabilities.
 
 when not defined(nimcore):
@@ -22,6 +28,7 @@ import compiler / [options, commands, modules, sem,
   idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper,
   pathutils]
 
+
 when defined(windows):
   import winlean
 else:
@@ -43,6 +50,8 @@ Options:
   --debug                 enable debug output
   --log                   enable verbose logging to nimsuggest.log file
   --v1                    use version 1 of the protocol; for backwards compatibility
+  --v2                    use version 2(default) of the protocol
+  --v3                    use version 3 of the protocol
   --refresh               perform automatic refreshes to keep the analysis precise
   --maxresults:N          limit the number of suggestions to N
   --tester                implies --stdin and outputs a line
@@ -76,6 +85,9 @@ var
   requests: Channel[string]
   results: Channel[Suggest]
 
+proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int;
+  graph: ModuleGraph);
+
 proc writelnToChannel(line: string) =
   results.send(Suggest(section: ideMsg, doc: line))
 
@@ -137,7 +149,7 @@ proc listEpc(): SexpNode =
     argspecs = sexp("file line column dirtyfile".split(" ").map(newSSymbol))
     docstring = sexp("line starts at 1, column at 0, dirtyfile is optional")
   result = newSList()
-  for command in ["sug", "con", "def", "use", "dus", "chk", "mod"]:
+  for command in ["sug", "con", "def", "use", "dus", "chk", "mod", "globalSymbols", "recompile", "saved", "chkFile"]:
     let
       cmd = sexp(command)
       methodDesc = newSList()
@@ -163,6 +175,11 @@ proc symFromInfo(graph: ModuleGraph; trackPos: TLineInfo): PSym =
 proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int;
              graph: ModuleGraph) =
   let conf = graph.config
+
+  if conf.suggestVersion == 3:
+    executeNoHooksV3(cmd, file, dirtyfile, line, col, graph)
+    return
+
   myLog("cmd: " & $cmd & ", file: " & file.string &
         ", dirtyFile: " & dirtyfile.string &
         "[" & $line & ":" & $col & "]")
@@ -436,6 +453,10 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) =
   of "terse": toggle optIdeTerse
   of "known": conf.ideCmd = ideKnown
   of "project": conf.ideCmd = ideProject
+  of "changed": conf.ideCmd = ideChanged
+  of "globalsymbols": conf.ideCmd = ideGlobalSymbols
+  of "chkfile": conf.ideCmd = ideChkFile
+  of "recompile": conf.ideCmd = ideRecompile
   else: err()
   var dirtyfile = ""
   var orig = ""
@@ -463,14 +484,23 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) =
     execute(conf.ideCmd, AbsoluteFile orig, AbsoluteFile dirtyfile, line, col, graph)
   sentinel()
 
+template benchmark(benchmarkName: string, code: untyped) =
+  block:
+    myLog "Started [" & benchmarkName & "]..."
+    let t0 = epochTime()
+    code
+    let elapsed = epochTime() - t0
+    let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 3)
+    myLog "CPU Time [" & benchmarkName & "] " & elapsedStr & "s"
+
 proc recompileFullProject(graph: ModuleGraph) =
-  #echo "recompiling full project"
-  resetSystemArtifacts(graph)
-  graph.vm = nil
-  graph.resetAllModules()
-  GC_fullCollect()
-  compileProject(graph)
-  #echo GC_getStatistics()
+  benchmark "Recompilation(clean)":
+    graph.resetForBackend()
+    graph.resetSystemArtifacts()
+    graph.vm = nil
+    graph.resetAllModules()
+    GC_fullCollect()
+    graph.compileProject()
 
 proc mainThread(graph: ModuleGraph) =
   let conf = graph.config
@@ -499,7 +529,7 @@ proc mainThread(graph: ModuleGraph) =
     else:
       os.sleep 250
       idle += 1
-    if idle == 20 and gRefresh:
+    if idle == 20 and gRefresh and conf.suggestVersion != 3:
       # we use some nimsuggest activity to enable a lazy recompile:
       conf.ideCmd = ideChk
       conf.writelnHook = proc (s: string) = discard
@@ -527,12 +557,18 @@ proc mainCommand(graph: ModuleGraph) =
 
   conf.setErrorMaxHighMaybe # honor --errorMax even if it may not make sense here
   # do not print errors, but log them
-  conf.writelnHook = myLog
-  conf.structuredErrorHook = nil
+  conf.writelnHook = proc (msg: string) = discard
+
+  if graph.config.suggestVersion == 3:
+    graph.config.structuredErrorHook = proc (conf: ConfigRef; info: TLineInfo; msg: string; sev: Severity) =
+      let suggest = Suggest(section: ideChk, filePath: toFullPath(conf, info),
+        line: toLinenumber(info), column: toColumn(info), doc: msg, forth: $sev)
+      graph.suggestErrors.mgetOrPut(info.fileIndex, @[]).add suggest
 
   # compile the project before showing any input so that we already
   # can answer questions right away:
-  compileProject(graph)
+  benchmark "Initial compilation":
+    compileProject(graph)
 
   open(requests)
   open(results)
@@ -584,8 +620,9 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string; conf: ConfigRef) =
         gMode = mepc
         conf.verbosity = 0          # Port number gotta be first.
       of "debug": incl(conf.globalOptions, optIdeDebug)
-      of "v2": conf.suggestVersion = 0
       of "v1": conf.suggestVersion = 1
+      of "v2": conf.suggestVersion = 0
+      of "v3": conf.suggestVersion = 3
       of "tester":
         gMode = mstdin
         gEmitEof = true
@@ -647,6 +684,182 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
   if self.loadConfigsAndProcessCmdLine(cache, conf, graph):
     mainCommand(graph)
 
+# v3 start
+
+proc recompilePartially(graph: ModuleGraph, projectFileIdx = InvalidFileIdx) =
+  if projectFileIdx == InvalidFileIdx:
+    myLog "Recompiling partially from root"
+  else:
+    myLog fmt "Recompiling partially starting from {graph.getModule(projectFileIdx)}"
+
+  # inst caches are breaking incremental compilation when the cache caches stuff
+  # from dirty buffer
+  # TODO: investigate more efficient way to achieve the same
+  # graph.typeInstCache.clear()
+  # graph.procInstCache.clear()
+
+  GC_fullCollect()
+
+  try:
+    benchmark "Recompilation":
+      graph.compileProject(projectFileIdx)
+  except Exception as e:
+    myLog fmt "Failed to recompile partially with the following error:\n {e.msg} \n\n {e.getStackTrace()}"
+    try:
+      graph.recompileFullProject()
+    except Exception as e:
+      myLog fmt "Failed clean recompilation:\n {e.msg} \n\n {e.getStackTrace()}"
+
+proc fileSymbols(graph: ModuleGraph, fileIdx: FileIndex): seq[tuple[sym: PSym, info: TLineInfo]] =
+  result = graph.suggestSymbols.getOrDefault(fileIdx, @[]).deduplicate
+
+proc findSymData(graph: ModuleGraph, file: AbsoluteFile; line, col: int):
+    tuple[sym: PSym, info: TLineInfo] =
+  let
+    fileIdx = fileInfoIdx(graph.config, file)
+    trackPos = newLineInfo(fileIdx, line, col)
+  for (sym, info) in graph.fileSymbols(fileIdx):
+    if isTracked(info, trackPos, sym.name.s.len):
+      return (sym, info)
+
+proc markDirtyIfNeeded(graph: ModuleGraph, file: string, originalFileIdx: FileIndex) =
+  let sha = $sha1.secureHashFile(file)
+  if graph.config.m.fileInfos[originalFileIdx.int32].hash != sha or graph.config.ideCmd == ideSug:
+    myLog fmt "{file} changed compared to last compilation"
+    graph.markDirty originalFileIdx
+    graph.markClientsDirty originalFileIdx
+  else:
+    myLog fmt "No changes in file {file} compared to last compilation"
+
+proc suggestResult(graph: ModuleGraph, sym: PSym, info: TLineInfo, defaultSection = ideNone) =
+  let section = if defaultSection != ideNone:
+                  defaultSection
+                elif sym.info == info:
+                  ideDef
+                else:
+                  ideUse
+  let suggest = symToSuggest(graph, sym, isLocal=false, section,
+                             info, 100, PrefixMatch.None, false, 0)
+  suggestResult(graph.config, suggest)
+
+const
+  # kinds for ideOutline and ideGlobalSymbols
+  searchableSymKinds = {skField, skEnumField, skIterator, skMethod, skFunc, skProc, skConverter, skTemplate}
+
+proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int;
+    graph: ModuleGraph) =
+  let conf = graph.config
+  conf.writelnHook = proc (s: string) = discard
+  conf.structuredErrorHook = proc (conf: ConfigRef; info: TLineInfo;
+                                   msg: string; sev: Severity) =
+    let suggest = Suggest(section: ideChk, filePath: toFullPath(conf, info),
+      line: toLinenumber(info), column: toColumn(info), doc: msg, forth: $sev)
+    graph.suggestErrors.mgetOrPut(info.fileIndex, @[]).add suggest
+
+  conf.ideCmd = cmd
+
+  myLog fmt "cmd: {cmd}, file: {file}[{line}:{col}], dirtyFile: {dirtyfile}"
+
+  var fileIndex: FileIndex
+
+  if not (cmd in {ideRecompile, ideGlobalSymbols}):
+    if not fileInfoKnown(conf, file):
+      myLog fmt "{file} is unknown, returning no results"
+      return
+
+    fileIndex = fileInfoIdx(conf, file)
+    msgs.setDirtyFile(
+      conf,
+      fileIndex,
+      if dirtyfile.isEmpty: AbsoluteFile"" else: dirtyfile)
+
+    if not dirtyfile.isEmpty:
+      graph.markDirtyIfNeeded(dirtyFile.string, fileInfoIdx(conf, file))
+
+  # these commands require fully compiled project
+  if cmd in {ideUse, ideDus, ideGlobalSymbols, ideChk} and graph.needsCompilation():
+    graph.recompilePartially()
+    # when doing incremental build for the project root we should make sure that
+    # everything is unmarked as no longer beeing dirty in case there is no
+    # longer reference to a particular module. E. g. A depends on B, B is marked
+    # as dirty and A loses B import.
+    graph.unmarkAllDirty()
+
+  # these commands require partially compiled project
+  elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile} and
+       (graph.needsCompilation(fileIndex) or cmd == ideSug):
+    # for ideSug use v2 implementation
+    if cmd == ideSug:
+      conf.m.trackPos = newLineInfo(fileIndex, line, col)
+      conf.m.trackPosAttached = false
+    else:
+      conf.m.trackPos = default(TLineInfo)
+
+    graph.recompilePartially(fileIndex)
+
+  case cmd
+  of ideDef:
+    let (sym, info) = graph.findSymData(file, line, col)
+    if sym != nil:
+      graph.suggestResult(sym, sym.info)
+  of ideUse, ideDus:
+    let symbol = graph.findSymData(file, line, col).sym
+    if symbol != nil:
+      for (sym, info) in graph.suggestSymbolsIter:
+        if sym == symbol:
+          graph.suggestResult(sym, info)
+  of ideHighlight:
+    let sym = graph.findSymData(file, line, col).sym
+    if sym != nil:
+      let usages = graph.fileSymbols(fileIndex).filterIt(it.sym == sym)
+      myLog fmt "Found {usages.len} usages in {file.string}"
+      for (sym, info) in usages:
+        graph.suggestResult(sym, info)
+  of ideRecompile:
+    graph.recompileFullProject()
+  of ideChanged:
+    graph.markDirtyIfNeeded(file.string, fileIndex)
+  of ideSug:
+    # ideSug performs partial build of the file, thus mark it dirty for the
+    # future calls.
+    graph.markDirtyIfNeeded(file.string, fileIndex)
+  of ideOutline:
+    let
+      module = graph.getModule fileIndex
+      symbols = graph.fileSymbols(fileIndex)
+        .filterIt(it.sym.info == it.info and
+                    (it.sym.owner == module or
+                     it.sym.kind in searchableSymKinds))
+    for (sym, _) in symbols:
+      suggestResult(
+        conf,
+        symToSuggest(graph, sym, false,
+                     ideOutline, sym.info, 100, PrefixMatch.None, false, 0))
+  of ideChk:
+    myLog fmt "Reporting errors for {graph.suggestErrors.len} file(s)"
+    for sug in graph.suggestErrorsIter:
+      suggestResult(graph.config, sug)
+  of ideChkFile:
+    let errors = graph.suggestErrors.getOrDefault(fileIndex, @[])
+    myLog fmt "Reporting {errors.len} error(s) for {file.string}"
+    for error in errors:
+      suggestResult(graph.config, error)
+  of ideGlobalSymbols:
+    var counter = 0
+    for (sym, info) in graph.suggestSymbolsIter:
+      if sfGlobal in sym.flags or sym.kind in searchableSymKinds:
+        if contains(sym.name.s, file.string):
+          inc counter
+          suggestResult(conf,
+                        symToSuggest(graph, sym, isLocal=false,
+                                     ideDef, info, 100, PrefixMatch.None, false, 0))
+        # stop after first 100 results
+        if counter > 100:
+          break
+  else:
+    myLog fmt "Discarding {cmd}"
+
+# v3 end
 when isMainModule:
   handleCmdLine(newIdentCache(), newConfigRef())
 else:
@@ -726,8 +939,9 @@ else:
     if self.loadConfigsAndProcessCmdLine(cache, conf, graph):
       mockCommand(graph)
     if gLogging:
+      log("Search paths:")
       for it in conf.searchPaths:
-        log(it.string)
+        log(" " & it.string)
 
     retval.doStopCompile = proc (): bool = false
     return NimSuggest(graph: retval, idle: 0, cachedMsgs: @[])

nimsuggest/tester.nim

@@ -252,7 +252,10 @@ proc runEpcTest(filename: string): int =
   for cmd in s.startup:
     if not runCmd(cmd, s.dest):
       quit "invalid command: " & cmd
-  let epccmd = s.cmd.replace("--tester", "--epc --v2 --log")
+  let epccmd = if s.cmd.contains("--v3"):
+    s.cmd.replace("--tester", "--epc --log")
+  else:
+    s.cmd.replace("--tester", "--epc --v2 --log")
   let cl = parseCmdLine(epccmd)
   var p = startProcess(command=cl[0], args=cl[1 .. ^1],
                        options={poStdErrToStdOut, poUsePath,

nimsuggest/tests/tv3.nim

@@ -0,0 +1,25 @@
+# tests v3
+
+type
+  Foo* = ref object of RootObj
+    bar*: string
+
+proc test(f: Foo) =
+  echo f.ba#[!]#r
+
+discard """
+$nimsuggest --v3 --tester $file
+>use $1
+def	skField	tv3.Foo.bar	string	$file	5	4	""	100
+use	skField	tv3.Foo.bar	string	$file	8	9	""	100
+>def $1
+def	skField	tv3.Foo.bar	string	$file	5	4	""	100
+>outline $1
+outline	skType	tv3.Foo	Foo	$file	4	2	""	100
+outline	skField	tv3.Foo.bar	string	$file	5	4	""	100
+outline	skProc	tv3.test	proc (f: Foo){.gcsafe, locks: 0.}	$file	7	5	""	100
+>sug $1
+sug	skField	bar	string	$file	5	4	""	100	Prefix
+>globalSymbols test
+def	skProc	tv3.test	proc (f: Foo){.gcsafe, locks: 0.}	$file	7	5	""	100
+"""