new dependency tracking for nimsuggest

2026-04-20 22:35:24 +00:00 · 2016-11-05 01:19:23 +01:00
parent 33ebf3e5fb
commit 9e6fb3f696
21 changed files with 284 additions and 393 deletions
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -16,6 +16,8 @@ import
  condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
  lowerings, semparallel

+from modulegraphs import ModuleGraph
+
 import strutils except `%` # collides with ropes.`%`

 when options.hasTinyCBackend:
@@ -1174,7 +1176,7 @@ proc newModule(module: PSym): BModule =
    if (sfDeadCodeElim in module.flags):
      internalError("added pending module twice: " & module.filename)

-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
  result = newModule(module)
  if optGenIndex in gGlobalOptions and generatedHeader == nil:
    let f = if headerFile.len > 0: headerFile else: gProjectFull
@@ -1209,7 +1211,7 @@ proc getCFile(m: BModule): string =
      else: ".c"
  result = changeFileExt(completeCFilePath(m.cfilename.withPackageName), ext)

-proc myOpenCached(module: PSym, rd: PRodReader): PPassContext =
+proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
  assert optSymbolFiles in gGlobalOptions
  var m = newModule(module)
  readMergeInfo(getCFile(m), m)
--- a/compiler/depends.nim
+++ b/compiler/depends.nim
@@ -12,6 +12,8 @@
 import
  os, options, ast, astalgo, msgs, ropes, idents, passes, importer

+from modulegraphs import ModuleGraph
+
 proc generateDot*(project: string)

 type
@@ -46,7 +48,7 @@ proc generateDot(project: string) =
      rope(changeFileExt(extractFilename(project), "")), gDotGraph],
            changeFileExt(project, "dot"))

-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
  var g: PGen
  new(g)
  g.module = module
--- a/compiler/docgen2.nim
+++ b/compiler/docgen2.nim
@@ -13,6 +13,8 @@
 import
  os, options, ast, astalgo, msgs, ropes, idents, passes, docgen

+from modulegraphs import ModuleGraph
+
 type
  TGen = object of TPassContext
    doc: PDoc
@@ -49,7 +51,7 @@ proc processNodeJson(c: PPassContext, n: PNode): PNode =
  var g = PGen(c)
  generateJson(g.doc, n)

-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
  var g: PGen
  new(g)
  g.module = module
--- a/compiler/importer.nim
+++ b/compiler/importer.nim
@@ -162,7 +162,7 @@ proc importModuleAs(n: PNode, realModule: PSym): PSym =
 proc myImportModule(c: PContext, n: PNode): PSym =
  var f = checkModuleName(n)
  if f != InvalidFileIDX:
-    result = importModuleAs(n, gImportModule(c.module, f, c.cache))
+    result = importModuleAs(n, gImportModule(c.graph, c.module, f, c.cache))
    # we cannot perform this check reliably because of
    # test: modules/import_in_config)
    if result.info.fileIndex == c.module.info.fileIndex and
--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -35,6 +35,8 @@ import
  times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils,
  intsets, cgmeth, lowerings

+from modulegraphs import ModuleGraph
+
 type
  TTarget = enum
    targetJS, targetPHP
@@ -2272,11 +2274,11 @@ proc myClose(b: PPassContext, n: PNode): PNode =
    for obj, content in items(globals.classes):
      genClass(obj, content, ext)

-proc myOpenCached(s: PSym, rd: PRodReader): PPassContext =
+proc myOpenCached(graph: ModuleGraph; s: PSym, rd: PRodReader): PPassContext =
  internalError("symbol files are not possible with the JS code generator")
  result = nil

-proc myOpen(s: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext =
  var r = newModule(s)
  r.target = if gCmd == cmdCompileToPHP: targetPHP else: targetJS
  result = r
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -15,7 +15,8 @@ import
  wordrecg, sem, semdata, idents, passes, docgen, extccomp,
  cgen, jsgen, json, nversion,
  platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
-  docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists
+  docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists,
+  modulegraphs

 from magicsys import systemModule, resetSysTypes

@@ -30,80 +31,44 @@ proc semanticPasses =
  registerPass verbosePass
  registerPass semPass

-proc commandGenDepend(cache: IdentCache) =
+proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) =
  semanticPasses()
  registerPass(gendependPass)
  registerPass(cleanupPass)
-  compileProject(cache)
+  compileProject(graph, cache)
  generateDot(gProjectFull)
  execExternalProgram("dot -Tpng -o" & changeFileExt(gProjectFull, "png") &
      ' ' & changeFileExt(gProjectFull, "dot"))

-proc commandCheck(cache: IdentCache) =
+proc commandCheck(graph: ModuleGraph; cache: IdentCache) =
  msgs.gErrorMax = high(int)  # do not stop after first error
  defineSymbol("nimcheck")
  semanticPasses()            # use an empty backend for semantic checking only
  rodPass()
-  compileProject(cache)
+  compileProject(graph, cache)

-proc commandDoc2(cache: IdentCache; json: bool) =
+proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) =
  msgs.gErrorMax = high(int)  # do not stop after first error
  semanticPasses()
  if json: registerPass(docgen2JsonPass)
  else: registerPass(docgen2Pass)
  #registerPass(cleanupPass())
-  compileProject(cache)
+  compileProject(graph, cache)
  finishDoc2Pass(gProjectName)

-proc commandCompileToC(cache: IdentCache) =
+proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) =
  extccomp.initVars()
  semanticPasses()
  registerPass(cgenPass)
  rodPass()
  #registerPass(cleanupPass())

-  compileProject(cache)
+  compileProject(graph, cache)
  cgenWriteModules()
  if gCmd != cmdRun:
    extccomp.callCCompiler(changeFileExt(gProjectFull, ""))

-  if isServing:
-    # caas will keep track only of the compilation commands
-    lastCaasCmd = curCaasCmd
-    resetCgenModules()
-    for i in 0 .. <gMemCacheData.len:
-      gMemCacheData[i].hashStatus = hashCached
-      gMemCacheData[i].needsRecompile = Maybe
-
-      # XXX: clean these global vars
-      # ccgstmts.gBreakpoints
-      # ccgthreadvars.nimtv
-      # ccgthreadvars.nimtVDeps
-      # ccgthreadvars.nimtvDeclared
-      # cgendata
-      # cgmeth?
-      # condsyms?
-      # depends?
-      # lexer.gLinesCompiled
-      # msgs - error counts
-      # magicsys, when system.nim changes
-      # rodread.rodcompilerProcs
-      # rodread.gTypeTable
-      # rodread.gMods
-
-      # !! ropes.cache
-      # semthreads.computed?
-      #
-      # suggest.usageSym
-      #
-      # XXX: can we run out of IDs?
-      # XXX: detect config reloading (implement as error/require restart)
-      # XXX: options are appended (they will accumulate over time)
-    resetCompilationLists()
-    ccgutils.resetCaches()
-    GC_fullCollect()
-
-proc commandCompileToJS(cache: IdentCache) =
+proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) =
  #incl(gGlobalOptions, optSafeCode)
  setTarget(osJS, cpuJS)
  #initDefines()
@@ -113,9 +78,9 @@ proc commandCompileToJS(cache: IdentCache) =
  if gCmd == cmdCompileToPHP: defineSymbol("nimphp")
  semanticPasses()
  registerPass(JSgenPass)
-  compileProject(cache)
+  compileProject(graph, cache)

-proc interactivePasses(cache: IdentCache) =
+proc interactivePasses(graph: ModuleGraph; cache: IdentCache) =
  #incl(gGlobalOptions, optSafeCode)
  #setTarget(osNimrodVM, cpuNimrodVM)
  initDefines()
@@ -125,28 +90,28 @@ proc interactivePasses(cache: IdentCache) =
  registerPass(semPass)
  registerPass(evalPass)

-proc commandInteractive(cache: IdentCache) =
+proc commandInteractive(graph: ModuleGraph; cache: IdentCache) =
  msgs.gErrorMax = high(int)  # do not stop after first error
-  interactivePasses(cache)
-  compileSystemModule(cache)
+  interactivePasses(graph, cache)
+  compileSystemModule(graph, cache)
  if commandArgs.len > 0:
-    discard compileModule(fileInfoIdx(gProjectFull), cache, {})
+    discard graph.compileModule(fileInfoIdx(gProjectFull), cache, {})
  else:
-    var m = makeStdinModule()
+    var m = graph.makeStdinModule()
    incl(m.flags, sfMainModule)
-    processModule(m, llStreamOpenStdIn(), nil, cache)
+    processModule(graph, m, llStreamOpenStdIn(), nil, cache)

 const evalPasses = [verbosePass, semPass, evalPass]

-proc evalNim(nodes: PNode, module: PSym; cache: IdentCache) =
-  carryPasses(nodes, module, cache, evalPasses)
+proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym; cache: IdentCache) =
+  carryPasses(graph, nodes, module, cache, evalPasses)

-proc commandEval(cache: IdentCache; exp: string) =
+proc commandEval(graph: ModuleGraph; cache: IdentCache; exp: string) =
  if systemModule == nil:
-    interactivePasses(cache)
-    compileSystemModule(cache)
+    interactivePasses(graph, cache)
+    compileSystemModule(graph, cache)
  let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
-  evalNim(echoExp.parseString(cache), makeStdinModule(), cache)
+  evalNim(graph, echoExp.parseString(cache), makeStdinModule(graph), cache)

 proc commandScan(cache: IdentCache) =
  var f = addFileExt(mainCommandArg(), NimExt)
@@ -165,75 +130,11 @@ proc commandScan(cache: IdentCache) =
  else:
    rawMessage(errCannotOpenFile, f)

-proc commandSuggest(cache: IdentCache) =
-  if isServing:
-    # XXX: hacky work-around ahead
-    # Currently, it's possible to issue a idetools command, before
-    # issuing the first compile command. This will leave the compiler
-    # cache in a state where "no recompilation is necessary", but the
-    # cgen pass was never executed at all.
-    commandCompileToC(cache)
-    let gDirtyBufferIdx = gTrackPos.fileIndex
-    discard compileModule(gDirtyBufferIdx, cache, {sfDirty})
-    resetModule(gDirtyBufferIdx)
-  else:
-    msgs.gErrorMax = high(int)  # do not stop after first error
-    semanticPasses()
-    rodPass()
-    # XXX: this handles the case when the dirty buffer is the main file,
-    # but doesn't handle the case when it's imported module
-    #var projFile = if gProjectMainIdx == gDirtyOriginalIdx: gDirtyBufferIdx
-    #               else: gProjectMainIdx
-    compileProject(cache) #(projFile)
-
-proc resetMemory =
-  resetCompilationLists()
-  ccgutils.resetCaches()
-  resetAllModules()
-  resetRopeCache()
-  resetSysTypes()
-  gOwners = @[]
-  resetIdentCache()
-
-  # XXX: clean these global vars
-  # ccgstmts.gBreakpoints
-  # ccgthreadvars.nimtv
-  # ccgthreadvars.nimtVDeps
-  # ccgthreadvars.nimtvDeclared
-  # cgendata
-  # cgmeth?
-  # condsyms?
-  # depends?
-  # lexer.gLinesCompiled
-  # msgs - error counts
-  # magicsys, when system.nim changes
-  # rodread.rodcompilerProcs
-  # rodread.gTypeTable
-  # rodread.gMods
-
-  # !! ropes.cache
-  #
-  # suggest.usageSym
-  #
-  # XXX: can we run out of IDs?
-  # XXX: detect config reloading (implement as error/require restart)
-  # XXX: options are appended (they will accumulate over time)
-  # vis = visimpl
-  when compileOption("gc", "v2"):
-    gcDebugging = true
-    echo "COLLECT 1"
-    GC_fullCollect()
-    echo "COLLECT 2"
-    GC_fullCollect()
-    echo "COLLECT 3"
-    GC_fullCollect()
-    echo GC_getStatistics()
-
 const
  SimulateCaasMemReset = false
  PrintRopeCacheStats = false

-proc mainCommand*(cache: IdentCache) =
+proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
  when SimulateCaasMemReset:
    gGlobalOptions.incl(optCaasEnabled)

@@ -249,28 +150,28 @@ proc mainCommand*(cache: IdentCache) =
  of "c", "cc", "compile", "compiletoc":
    # compile means compileToC currently
    gCmd = cmdCompileToC
-    commandCompileToC(cache)
+    commandCompileToC(graph, cache)
  of "cpp", "compiletocpp":
    gCmd = cmdCompileToCpp
    defineSymbol("cpp")
-    commandCompileToC(cache)
+    commandCompileToC(graph, cache)
  of "objc", "compiletooc":
    gCmd = cmdCompileToOC
    defineSymbol("objc")
-    commandCompileToC(cache)
+    commandCompileToC(graph, cache)
  of "run":
    gCmd = cmdRun
    when hasTinyCBackend:
      extccomp.setCC("tcc")
-      commandCompileToC(cache)
+      commandCompileToC(graph, cache)
    else:
      rawMessage(errInvalidCommandX, command)
  of "js", "compiletojs":
    gCmd = cmdCompileToJS
-    commandCompileToJS(cache)
+    commandCompileToJS(graph, cache)
  of "php":
    gCmd = cmdCompileToPHP
-    commandCompileToJS(cache)
+    commandCompileToJS(graph, cache)
  of "doc":
    wantMainModule()
    gCmd = cmdDoc
@@ -280,7 +181,7 @@ proc mainCommand*(cache: IdentCache) =
    gCmd = cmdDoc
    loadConfigs(DocConfig, cache)
    defineSymbol("nimdoc")
-    commandDoc2(cache, false)
+    commandDoc2(graph, cache, false)
  of "rst2html":
    gCmd = cmdRst2html
    loadConfigs(DocConfig, cache)
@@ -301,14 +202,14 @@ proc mainCommand*(cache: IdentCache) =
    loadConfigs(DocConfig, cache)
    wantMainModule()
    defineSymbol("nimdoc")
-    commandDoc2(cache, true)
+    commandDoc2(graph, cache, true)
  of "buildindex":
    gCmd = cmdDoc
    loadConfigs(DocConfig, cache)
    commandBuildIndex()
  of "gendepend":
    gCmd = cmdGenDepend
-    commandGenDepend(cache)
+    commandGenDepend(graph, cache)
  of "dump":
    gCmd = cmdDump
    if getConfigVar("dump.format") == "json":
@@ -337,7 +238,7 @@ proc mainCommand*(cache: IdentCache) =
      for it in iterSearchPath(searchPaths): msgWriteln(it)
  of "check":
    gCmd = cmdCheck
-    commandCheck(cache)
+    commandCheck(graph, cache)
  of "parse":
    gCmd = cmdParse
    wantMainModule()
@@ -346,26 +247,12 @@ proc mainCommand*(cache: IdentCache) =
    gCmd = cmdScan
    wantMainModule()
    commandScan(cache)
-    msgWriteln("Beware: Indentation tokens depend on the parser\'s state!")
+    msgWriteln("Beware: Indentation tokens depend on the parser's state!")
  of "secret":
    gCmd = cmdInteractive
-    commandInteractive(cache)
+    commandInteractive(graph, cache)
  of "e":
-    # XXX: temporary command for easier testing
-    commandEval(cache, mainCommandArg())
-  of "reset":
-    resetMemory()
-  of "idetools":
-    gCmd = cmdIdeTools
-    if gEvalExpr != "":
-      commandEval(cache, gEvalExpr)
-    else:
-      commandSuggest(cache)
-  of "serve":
-    isServing = true
-    gGlobalOptions.incl(optCaasEnabled)
-    msgs.gErrorMax = high(int)  # do not stop after first error
-    serve(cache, mainCommand)
+    commandEval(graph, cache, mainCommandArg())
  of "nop", "help":
    # prevent the "success" message:
    gCmd = cmdDump
@@ -393,4 +280,4 @@ proc mainCommand*(cache: IdentCache) =

  resetAttributes()

-proc mainCommand*() = mainCommand(newIdentCache())
+proc mainCommand*() = mainCommand(newModuleGraph(), newIdentCache())
--- a/compiler/modules.nim
+++ b/compiler/modules.nim
@@ -7,130 +7,118 @@
 #    distribution, for details about the copyright.
 #

-## implements the module handling
+## Implements the module handling, including the caching of modules.

 import
  ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options,
-  idents, os, lexer, idgen, passes, syntaxes, llstream
+  idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs

-type
-  TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
-  THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged
+when false:
+  type
+    TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
+    THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged

-  TModuleInMemory* = object
-    compiledAt*: float
-    hash*: SecureHash
-    deps*: seq[int32] ## XXX: slurped files are currently not tracked
-    needsRecompile*: TNeedRecompile
-    hashStatus*: THashStatus
+    TModuleInMemory* = object
+      hash*: SecureHash
+      deps*: seq[int32] ## XXX: slurped files are currently not tracked

-var
-  gCompiledModules: seq[PSym] = @[]
-  gMemCacheData*: seq[TModuleInMemory] = @[]
-    ## XXX: we should implement recycling of file IDs
-    ## if the user keeps renaming modules, the file IDs will keep growing
-  gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why.
-  packageSyms: TStrTable
+      needsRecompile*: TNeedRecompile
+      hashStatus*: THashStatus

-initStrTable(packageSyms)
+  var
+    gCompiledModules: seq[PSym] = @[]
+    gMemCacheData*: seq[TModuleInMemory] = @[]
+      ## XXX: we should implement recycling of file IDs
+      ## if the user keeps renaming modules, the file IDs will keep growing
+    gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why.

-proc getModule*(fileIdx: int32): PSym =
-  if fileIdx >= 0 and fileIdx < gCompiledModules.len:
-    result = gCompiledModules[fileIdx]
+  proc hashChanged(fileIdx: int32): bool =
+    internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len

-proc hashChanged(fileIdx: int32): bool =
-  internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
+    template updateStatus =
+      gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
+                                         else: hashNotChanged
+      # echo "TESTING Hash: ", fileIdx.toFilename, " ", result

-  template updateStatus =
-    gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
-                                       else: hashNotChanged
-    # echo "TESTING Hash: ", fileIdx.toFilename, " ", result
+    case gMemCacheData[fileIdx].hashStatus
+    of hashHasChanged:
+      result = true
+    of hashNotChanged:
+      result = false
+    of hashCached:
+      let newHash = secureHashFile(fileIdx.toFullPath)
+      result = newHash != gMemCacheData[fileIdx].hash
+      gMemCacheData[fileIdx].hash = newHash
+      updateStatus()
+    of hashNotTaken:
+      gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
+      result = true
+      updateStatus()

-  case gMemCacheData[fileIdx].hashStatus
-  of hashHasChanged:
-    result = true
-  of hashNotChanged:
-    result = false
-  of hashCached:
-    let newHash = secureHashFile(fileIdx.toFullPath)
-    result = newHash != gMemCacheData[fileIdx].hash
-    gMemCacheData[fileIdx].hash = newHash
-    updateStatus()
-  of hashNotTaken:
-    gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
-    result = true
-    updateStatus()
+  proc doHash(fileIdx: int32) =
+    if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
+      # echo "FIRST Hash: ", fileIdx.ToFilename
+      gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)

-proc doHash(fileIdx: int32) =
-  if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
-    # echo "FIRST Hash: ", fileIdx.ToFilename
-    gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
+  proc resetModule*(fileIdx: int32) =
+    # echo "HARD RESETTING ", fileIdx.toFilename
+    if fileIdx <% gMemCacheData.len:
+      gMemCacheData[fileIdx].needsRecompile = Yes
+    if fileIdx <% gCompiledModules.len:
+      gCompiledModules[fileIdx] = nil
+    if fileIdx <% cgendata.gModules.len:
+      cgendata.gModules[fileIdx] = nil

-proc addDep(x: PSym, dep: int32) =
-  growCache gMemCacheData, dep
-  gMemCacheData[x.position].deps.safeAdd(dep)
+  proc resetModule*(module: PSym) =
+    let conflict = getModule(module.position.int32)
+    if conflict == nil: return
+    doAssert conflict == module
+    resetModule(module.position.int32)
+    initStrTable(module.tab)

-proc resetModule*(fileIdx: int32) =
-  # echo "HARD RESETTING ", fileIdx.toFilename
-  if fileIdx <% gMemCacheData.len:
-    gMemCacheData[fileIdx].needsRecompile = Yes
-  if fileIdx <% gCompiledModules.len:
-    gCompiledModules[fileIdx] = nil
-  if fileIdx <% cgendata.gModules.len:
-    cgendata.gModules[fileIdx] = nil
+  proc resetAllModules* =
+    for i in 0..gCompiledModules.high:
+      if gCompiledModules[i] != nil:
+        resetModule(i.int32)
+    resetPackageCache()
+    # for m in cgenModules(): echo "CGEN MODULE FOUND"

-proc resetModule*(module: PSym) =
-  let conflict = getModule(module.position.int32)
-  if conflict == nil: return
-  doAssert conflict == module
-  resetModule(module.position.int32)
-  initStrTable(module.tab)
+  proc resetAllModulesHard* =
+    resetPackageCache()
+    gCompiledModules.setLen 0
+    gMemCacheData.setLen 0
+    magicsys.resetSysTypes()
+    # XXX
+    #gOwners = @[]

-proc resetAllModules* =
-  for i in 0..gCompiledModules.high:
-    if gCompiledModules[i] != nil:
-      resetModule(i.int32)
-  resetPackageCache()
-  initStrTable(packageSyms)
-  # for m in cgenModules(): echo "CGEN MODULE FOUND"
+  proc checkDepMem(fileIdx: int32): TNeedRecompile =
+    template markDirty =
+      resetModule(fileIdx)
+      return Yes

-proc resetAllModulesHard* =
-  resetPackageCache()
-  gCompiledModules.setLen 0
-  gMemCacheData.setLen 0
-  magicsys.resetSysTypes()
-  initStrTable(packageSyms)
-  # XXX
-  #gOwners = @[]
+    if gFuzzyGraphChecking:
+      if gMemCacheData[fileIdx].needsRecompile != Maybe:
+        return gMemCacheData[fileIdx].needsRecompile
+    else:
+      # cycle detection: We claim that a cycle does no harm.
+      if gMemCacheData[fileIdx].needsRecompile == Probing:
+        return No

-proc checkDepMem(fileIdx: int32): TNeedRecompile =
-  template markDirty =
-    resetModule(fileIdx)
-    return Yes
+    if optForceFullMake in gGlobalOptions or hashChanged(fileIdx):
+      markDirty()

-  if gFuzzyGraphChecking:
-    if gMemCacheData[fileIdx].needsRecompile != Maybe:
-      return gMemCacheData[fileIdx].needsRecompile
-  else:
-    # cycle detection: We claim that a cycle does no harm.
-    if gMemCacheData[fileIdx].needsRecompile == Probing:
-      return No
+    if gMemCacheData[fileIdx].deps != nil:
+      gMemCacheData[fileIdx].needsRecompile = Probing
+      for dep in gMemCacheData[fileIdx].deps:
+        let d = checkDepMem(dep)
+        if d in {Yes, Recompiled}:
+          # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
+          markDirty()

-  if optForceFullMake in gGlobalOptions or hashChanged(fileIdx):
-    markDirty()
+    gMemCacheData[fileIdx].needsRecompile = No
+    return No

-  if gMemCacheData[fileIdx].deps != nil:
-    gMemCacheData[fileIdx].needsRecompile = Probing
-    for dep in gMemCacheData[fileIdx].deps:
-      let d = checkDepMem(dep)
-      if d in {Yes, Recompiled}:
-        # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
-        markDirty()
-
-  gMemCacheData[fileIdx].needsRecompile = No
-  return No
-
-proc newModule(fileIdx: int32): PSym =
+proc newModule(graph: ModuleGraph; fileIdx: int32): PSym =
  # We cannot call ``newSym`` here, because we have to circumvent the ID
  # mechanism, which we do in order to assign each module a persistent ID.
  new(result)
@@ -143,20 +131,19 @@ proc newModule(fileIdx: int32): PSym =

  result.info = newLineInfo(fileIdx, 1, 1)
  let pack = getIdent(getPackageName(filename))
-  var packSym = packageSyms.strTableGet(pack)
+  var packSym = graph.packageSyms.strTableGet(pack)
  if packSym == nil:
    let pck = getPackageName(filename)
    let pck2 = if pck.len > 0: pck else: "unknown"
    packSym = newSym(skPackage, getIdent(pck2), nil, result.info)
    initStrTable(packSym.tab)
-    packageSyms.strTableAdd(packSym)
+    graph.packageSyms.strTableAdd(packSym)

  result.owner = packSym
  result.position = fileIdx

-  growCache gMemCacheData, fileIdx
-  growCache gCompiledModules, fileIdx
-  gCompiledModules[result.position] = result
+  growCache graph.modules, fileIdx
+  graph.modules[result.position] = result

  incl(result.flags, sfUsed)
  initStrTable(result.tab)
@@ -167,13 +154,12 @@ proc newModule(fileIdx: int32): PSym =
  # strTableIncl() for error corrections:
  discard strTableIncl(packSym.tab, result)

-proc compileModule*(fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
-  result = getModule(fileIdx)
+proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
+  result = graph.getModule(fileIdx)
  if result == nil:
-    growCache gMemCacheData, fileIdx
-    gMemCacheData[fileIdx].needsRecompile = Probing
-    result = newModule(fileIdx)
-    #var rd = handleSymbolFile(result)
+    #growCache gMemCacheData, fileIdx
+    #gMemCacheData[fileIdx].needsRecompile = Probing
+    result = newModule(graph, fileIdx)
    var rd: PRodReader
    result.flags = result.flags + flags
    if sfMainModule in result.flags:
@@ -182,44 +168,51 @@ proc compileModule*(fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
    if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
      rd = handleSymbolFile(result, cache)
      if result.id < 0:
-        internalError("handleSymbolFile should have set the module\'s ID")
+        internalError("handleSymbolFile should have set the module's ID")
        return
    else:
      result.id = getID()
-    let validFile = processModule(result,
+    discard processModule(graph, result,
      if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,
      rd, cache)
-    if optCaasEnabled in gGlobalOptions:
-      gMemCacheData[fileIdx].compiledAt = gLastCmdTime
-      gMemCacheData[fileIdx].needsRecompile = Recompiled
-      if validFile: doHash fileIdx
-  else:
-    if checkDepMem(fileIdx) == Yes:
-      result = compileModule(fileIdx, cache, flags)
-    else:
-      result = gCompiledModules[fileIdx]
+    #if optCaasEnabled in gGlobalOptions:
+    #  gMemCacheData[fileIdx].needsRecompile = Recompiled
+    #  if validFile: doHash fileIdx
+  elif graph.isDirty(result):
+    result.flags.excl sfDirty
+    # reset module fields:
+    initStrTable(result.tab)
+    result.ast = nil
+    discard processModule(graph, result,
+      if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,
+      nil, cache)
+    graph.markClientsDirty(fileIdx)
+    when false:
+      if checkDepMem(fileIdx) == Yes:
+        result = compileModule(fileIdx, cache, flags)
+      else:
+        result = gCompiledModules[fileIdx]

-proc importModule*(s: PSym, fileIdx: int32; cache: IdentCache): PSym {.procvar.} =
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: int32;
+                   cache: IdentCache): PSym {.procvar.} =
  # this is called by the semantic checking phase
-  result = compileModule(fileIdx, cache, {})
-  if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
+  result = compileModule(graph, fileIdx, cache, {})
+  graph.addDep(s, fileIdx)
  #if sfSystemModule in result.flags:
  #  localError(result.info, errAttemptToRedefine, result.name.s)
  # restore the notes for outer module:
  gNotes = if s.owner.id == gMainPackageId: gMainPackageNotes
           else: ForeignPackageNotes

-proc includeModule*(s: PSym, fileIdx: int32; cache: IdentCache): PNode {.procvar.} =
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: int32;
+                    cache: IdentCache): PNode {.procvar.} =
  result = syntaxes.parseFile(fileIdx, cache)
-  if optCaasEnabled in gGlobalOptions:
-    growCache gMemCacheData, fileIdx
-    addDep(s, fileIdx)
-    doHash(fileIdx)
+  graph.addDep(s, fileIdx)

-proc compileSystemModule*(cache: IdentCache) =
+proc compileSystemModule*(graph: ModuleGraph; cache: IdentCache) =
  if magicsys.systemModule == nil:
    systemFileIdx = fileInfoIdx(options.libpath/"system.nim")
-    discard compileModule(systemFileIdx, cache, {sfSystemModule})
+    discard graph.compileModule(systemFileIdx, cache, {sfSystemModule})

 proc wantMainModule* =
  if gProjectFull.len == 0:
@@ -229,18 +222,19 @@ proc wantMainModule* =
 passes.gIncludeFile = includeModule
 passes.gImportModule = importModule

-proc compileProject*(cache: IdentCache; projectFileIdx = -1'i32) =
+proc compileProject*(graph: ModuleGraph; cache: IdentCache;
+                     projectFileIdx = -1'i32) =
  wantMainModule()
  let systemFileIdx = fileInfoIdx(options.libpath / "system.nim")
  let projectFile = if projectFileIdx < 0: gProjectMainIdx else: projectFileIdx
  if projectFile == systemFileIdx:
-    discard compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
+    discard graph.compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
  else:
-    compileSystemModule(cache)
-    discard compileModule(projectFile, cache, {sfMainModule})
+    graph.compileSystemModule(cache)
+    discard graph.compileModule(projectFile, cache, {sfMainModule})

-proc makeModule*(filename: string): PSym =
-  result = newModule(fileInfoIdx filename)
+proc makeModule*(graph: ModuleGraph; filename: string): PSym =
+  result = graph.newModule(fileInfoIdx filename)
  result.id = getID()

-proc makeStdinModule*(): PSym = makeModule"stdin"
+proc makeStdinModule*(graph: ModuleGraph): PSym = graph.makeModule"stdin"
--- a/compiler/nim.nim
+++ b/compiler/nim.nim
@@ -14,14 +14,14 @@ when defined(gcc) and defined(windows):
    {.link: "icons/nim_icon.o".}

 when defined(amd64) and defined(windows) and defined(vcc):
-  {.link: "icons/nim-amd64-windows-vcc.res" .}
+  {.link: "icons/nim-amd64-windows-vcc.res".}
 when defined(i386) and defined(windows) and defined(vcc):
-  {.link: "icons/nim-i386-windows-vcc.res" .}
+  {.link: "icons/nim-i386-windows-vcc.res".}

 import
  commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
  extccomp, strutils, os, osproc, platform, main, parseopt, service,
-  nodejs, scriptconfig, idents
+  nodejs, scriptconfig, idents, modulegraphs

 when hasTinyCBackend:
  import tccgen
@@ -73,7 +73,7 @@ proc handleCmdLine(cache: IdentCache) =
    processCmdLine(passCmd2, "")
    if options.command == "":
      rawMessage(errNoCommand, command)
-    mainCommand(cache)
+    mainCommand(newModuleGraph(), cache)
    if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics())
    #echo(GC_getStatistics())
    if msgs.gErrorCounter == 0:
--- a/compiler/nimsuggest/nimsuggest.nim
+++ b/compiler/nimsuggest/nimsuggest.nim
@@ -1,12 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Nimsuggest has been moved to https://github.com/nim-lang/nimsuggest
-
-{.error: "This project has moved to the following repo: https://github.com/nim-lang/nimsuggest".}
--- a/compiler/passaux.nim
+++ b/compiler/passaux.nim
@@ -12,7 +12,9 @@
 import
  strutils, ast, astalgo, passes, idents, msgs, options, idgen

-proc verboseOpen(s: PSym; cache: IdentCache): PPassContext =
+from modulegraphs import ModuleGraph
+
+proc verboseOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext =
  #MessageOut('compiling ' + s.name.s);
  result = nil                # we don't need a context
  rawMessage(hintProcessing, s.name.s)
--- a/compiler/passes.nim
+++ b/compiler/passes.nim
@@ -13,7 +13,7 @@
 import
  strutils, lists, options, ast, astalgo, llstream, msgs, platform, os,
  condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
-  nimsets, syntaxes, times, rodread, idgen
+  nimsets, syntaxes, times, rodread, idgen, modulegraphs

 type
  TPassContext* = object of RootObj # the pass's context
@@ -21,9 +21,9 @@ type

  PPassContext* = ref TPassContext

-  TPassOpen* = proc (module: PSym; cache: IdentCache): PPassContext {.nimcall.}
+  TPassOpen* = proc (graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext {.nimcall.}
  TPassOpenCached* =
-    proc (module: PSym, rd: PRodReader): PPassContext {.nimcall.}
+    proc (graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext {.nimcall.}
  TPassClose* = proc (p: PPassContext, n: PNode): PNode {.nimcall.}
  TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}

@@ -48,8 +48,8 @@ proc makePass*(open: TPassOpen = nil,

 # the semantic checker needs these:
 var
-  gImportModule*: proc (m: PSym, fileIdx: int32; cache: IdentCache): PSym {.nimcall.}
-  gIncludeFile*: proc (m: PSym, fileIdx: int32; cache: IdentCache): PNode {.nimcall.}
+  gImportModule*: proc (graph: ModuleGraph; m: PSym, fileIdx: int32; cache: IdentCache): PSym {.nimcall.}
+  gIncludeFile*: proc (graph: ModuleGraph; m: PSym, fileIdx: int32; cache: IdentCache): PNode {.nimcall.}

 # implementation

@@ -90,29 +90,32 @@ proc registerPass*(p: TPass) =
  gPasses[gPassesLen] = p
  inc(gPassesLen)

-proc carryPass*(p: TPass, module: PSym; cache: IdentCache;
+proc carryPass*(g: ModuleGraph; p: TPass, module: PSym; cache: IdentCache;
                m: TPassData): TPassData =
-  var c = p.open(module, cache)
+  var c = p.open(g, module, cache)
  result.input = p.process(c, m.input)
  result.closeOutput = if p.close != nil: p.close(c, m.closeOutput)
                       else: m.closeOutput

-proc carryPasses*(nodes: PNode, module: PSym; cache: IdentCache; passes: TPasses) =
+proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym;
+                  cache: IdentCache; passes: TPasses) =
  var passdata: TPassData
  passdata.input = nodes
  for pass in passes:
-    passdata = carryPass(pass, module, cache, passdata)
+    passdata = carryPass(g, pass, module, cache, passdata)

-proc openPasses(a: var TPassContextArray, module: PSym; cache: IdentCache) =
+proc openPasses(g: ModuleGraph; a: var TPassContextArray;
+                module: PSym; cache: IdentCache) =
  for i in countup(0, gPassesLen - 1):
    if not isNil(gPasses[i].open):
-      a[i] = gPasses[i].open(module, cache)
+      a[i] = gPasses[i].open(g, module, cache)
    else: a[i] = nil

-proc openPassesCached(a: var TPassContextArray, module: PSym, rd: PRodReader) =
+proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym,
+                      rd: PRodReader) =
  for i in countup(0, gPassesLen - 1):
    if not isNil(gPasses[i].openCached):
-      a[i] = gPasses[i].openCached(module, rd)
+      a[i] = gPasses[i].openCached(g, module, rd)
      if a[i] != nil:
        a[i].fromCache = true
    else:
@@ -155,7 +158,7 @@ proc processImplicits(implicits: seq[string], nodeKind: TNodeKind,
    importStmt.addSon str
    if not processTopLevelStmt(importStmt, a): break

-proc processModule*(module: PSym, stream: PLLStream,
+proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
                    rd: PRodReader; cache: IdentCache): bool {.discardable.} =
  var
    p: TParsers
@@ -163,7 +166,7 @@ proc processModule*(module: PSym, stream: PLLStream,
    s: PLLStream
    fileIdx = module.fileIdx
  if rd == nil:
-    openPasses(a, module, cache)
+    openPasses(graph, a, module, cache)
    if stream == nil:
      let filename = fileIdx.toFullPathConsiderDirty
      s = llStreamOpen(filename, fmRead)
@@ -203,7 +206,7 @@ proc processModule*(module: PSym, stream: PLLStream,
    # id synchronization point for more consistent code generation:
    idSynchronizationPoint(1000)
  else:
-    openPassesCached(a, module, rd)
+    openPassesCached(graph, a, module, rd)
    var n = loadInitSection(rd)
    for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a)
    closePassesCached(a)
--- a/compiler/rodwrite.nim
+++ b/compiler/rodwrite.nim
@@ -16,6 +16,8 @@ import
  condsyms, ropes, idents, securehash, rodread, passes, importer, idgen,
  rodutils

+from modulegraphs import ModuleGraph
+
 type
  TRodWriter = object of TPassContext
    module: PSym
@@ -633,7 +635,7 @@ proc process(c: PPassContext, n: PNode): PNode =
  else:
    discard

-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
  if module.id < 0: internalError("rodwrite: module ID not set")
  var w = newRodWriter(module.fileIdx.getHash, module, cache)
  rawAddInterfaceSym(w, module)
--- a/compiler/scriptconfig.nim
+++ b/compiler/scriptconfig.nim
@@ -13,7 +13,7 @@
 import
  ast, modules, idents, passes, passaux, condsyms,
  options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs,
-  os, times, osproc, wordrecg, strtabs
+  os, times, osproc, wordrecg, strtabs, modulegraphs

 # we support 'cmpIgnoreStyle' natively for efficiency:
 from strutils import cmpIgnoreStyle, contains
@@ -134,9 +134,11 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext
  cbconf selfExe:
    setResult(a, os.getAppFilename())

-proc runNimScript*(cache: IdentCache; scriptName: string; freshDefines=true) =
+proc runNimScript*(cache: IdentCache; scriptName: string;
+                   freshDefines=true) =
  passes.gIncludeFile = includeModule
  passes.gImportModule = importModule
+  let graph = newModuleGraph()
  if freshDefines: initDefines()

  defineSymbol("nimscript")
@@ -146,15 +148,15 @@ proc runNimScript*(cache: IdentCache; scriptName: string; freshDefines=true) =

  appendStr(searchPaths, options.libpath)

-  var m = makeModule(scriptName)
+  var m = graph.makeModule(scriptName)
  incl(m.flags, sfMainModule)
  vm.globalCtx = setupVM(m, cache, scriptName)

-  compileSystemModule(cache)
-  discard processModule(m, llStreamOpen(scriptName, fmRead), nil, cache)
+  graph.compileSystemModule(cache)
+  discard graph.processModule(m, llStreamOpen(scriptName, fmRead), nil, cache)

  # ensure we load 'system.nim' again for the real non-config stuff!
-  resetAllModulesHard()
+  #resetAllModulesHard()
  vm.globalCtx = nil
  # do not remove the defined symbols
  #initDefines()
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -18,6 +18,8 @@ import
  evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity,
  semparallel, lowerings, pluginsupport, plugins.active

+from modulegraphs import ModuleGraph
+
 when defined(nimfix):
  import nimfix.prettybase

@@ -398,8 +400,8 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
        addSon(n, prc.ast)
  c.lastGenericIdx = c.generics.len

-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
-  var c = newContext(module, cache)
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
+  var c = newContext(graph, module, cache)
  if c.p != nil: internalError(module.info, "sem.myOpen")
  c.semConstExpr = semConstExpr
  c.semExpr = semExpr
@@ -428,8 +430,8 @@ proc myOpen(module: PSym; cache: IdentCache): PPassContext =
    gNotes = ForeignPackageNotes
  result = c

-proc myOpenCached(module: PSym; rd: PRodReader): PPassContext =
-  result = myOpen(module, rd.cache)
+proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext =
+  result = myOpen(graph, module, rd.cache)
  for m in items(rd.methods): methodDef(m, true)

 proc isImportSystemStmt(n: PNode): bool =
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2016 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -13,7 +13,8 @@ import
  strutils, lists, intsets, options, lexer, ast, astalgo, trees, treetab,
  wordrecg,
  ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math,
-  magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef
+  magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef,
+  modulegraphs

 type
  TOptionEntry* = object of lists.TListEntry # entries to put on a
@@ -107,6 +108,7 @@ type
                            op: TTypeAttachedOp; col: int): PSym {.nimcall.}
    selfName*: PIdent
    cache*: IdentCache
+    graph*: ModuleGraph
    signatures*: TStrTable

 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
@@ -151,7 +153,7 @@ proc newOptionEntry*(): POptionEntry =
  result.dynlib = nil
  result.notes = gNotes

-proc newContext*(module: PSym; cache: IdentCache): PContext =
+proc newContext*(graph: ModuleGraph; module: PSym; cache: IdentCache): PContext =
  new(result)
  result.ambiguousSymbols = initIntSet()
  initLinkedList(result.optionStack)
@@ -166,6 +168,7 @@ proc newContext*(module: PSym; cache: IdentCache): PContext =
  result.generics = @[]
  result.unknownIdents = initIntSet()
  result.cache = cache
+  result.graph = graph
  initStrTable(result.signatures)


--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -820,7 +820,7 @@ proc semAllTypeSections(c: PContext; n: PNode): PNode =
          if containsOrIncl(c.includedFiles, f):
            localError(n.info, errRecursiveDependencyX, f.toFilename)
          else:
-            let code = gIncludeFile(c.module, f, c.cache)
+            let code = gIncludeFile(c.graph, c.module, f, c.cache)
            gatherStmts c, code, result
            excl(c.includedFiles, f)
    of nkStmtList:
@@ -1418,7 +1418,7 @@ proc evalInclude(c: PContext, n: PNode): PNode =
      if containsOrIncl(c.includedFiles, f):
        localError(n.info, errRecursiveDependencyX, f.toFilename)
      else:
-        addSon(result, semStmt(c, gIncludeFile(c.module, f, c.cache)))
+        addSon(result, semStmt(c, gIncludeFile(c.graph, c.module, f, c.cache)))
        excl(c.includedFiles, f)

 proc setLine(n: PNode, info: TLineInfo) =
--- a/compiler/suggest.nim
+++ b/compiler/suggest.nim
@@ -233,7 +233,7 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
        # error: no known module name:
        typ = nil
      else:
-        let m = gImportModule(c.module, fullpath.fileInfoIdx, c.cache)
+        let m = gImportModule(c.graph, c.module, fullpath.fileInfoIdx, c.cache)
        if m == nil: typ = nil
        else:
          for it in items(n.sym.tab):
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -24,6 +24,8 @@ import
 from semfold import leValueConv, ordinalValToString
 from evaltempl import evalTemplate

+from modulegraphs import ModuleGraph
+
 when hasFFI:
  import evalffi

@@ -1516,7 +1518,7 @@ proc setupGlobalCtx(module: PSym; cache: IdentCache) =
  else:
    refresh(globalCtx, module)

-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
  #var c = newEvalContext(module, emRepl)
  #c.features = {allowCast, allowFFI, allowInfiniteLoops}
  #pushStackFrame(c, newStackFrame())
--- a/doc/basicopt.txt
+++ b/doc/basicopt.txt
@@ -5,7 +5,7 @@
 Command:
  //compile, c                compile project with default code generator (C)
  //doc                       generate the documentation for inputfile
-  //doc2                      generate the documentation for the whole project
+  //doc2                      generate the documentation for inputfile

 Arguments:
  arguments are passed to the program being run (if --run option is selected)
--- a/tools/nimsuggest/nimsuggest.nim
+++ b/tools/nimsuggest/nimsuggest.nim
@@ -17,7 +17,7 @@ import compiler/options, compiler/commands, compiler/modules, compiler/sem,
  compiler/passes, compiler/passaux, compiler/msgs, compiler/nimconf,
  compiler/extccomp, compiler/condsyms, compiler/lists,
  compiler/sigmatch, compiler/ast, compiler/scriptconfig,
-  compiler/idents
+  compiler/idents, compiler/modulegraphs

 when defined(windows):
  import winlean
@@ -128,19 +128,19 @@ proc findNode(n: PNode): PSym =
      let res = n.sons[i].findNode
      if res != nil: return res

-proc symFromInfo(gTrackPos: TLineInfo): PSym =
-  let m = getModule(gTrackPos.fileIndex)
+proc symFromInfo(graph: ModuleGraph; gTrackPos: TLineInfo): PSym =
+  let m = graph.getModule(gTrackPos.fileIndex)
  #echo m.isNil, " I knew it ", gTrackPos.fileIndex
  if m != nil and m.ast != nil:
    result = m.ast.findNode

 proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
-             cache: IdentCache) =
+             graph: ModuleGraph; cache: IdentCache) =
  if gLogging:
    logStr("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile & "[" & $line & ":" & $col & "]")
  gIdeCmd = cmd
  if cmd == ideUse and suggestVersion != 2:
-    modules.resetAllModules()
+    graph.resetAllModules()
  var isKnownFile = true
  let dirtyIdx = file.fileInfoIdx(isKnownFile)

@@ -152,23 +152,26 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
  if suggestVersion < 2:
    usageSym = nil
  if not isKnownFile:
-    compileProject(cache)
+    graph.compileProject(cache)
  if suggestVersion == 2 and gIdeCmd in {ideUse, ideDus} and
      dirtyfile.len == 0:
    discard "no need to recompile anything"
  else:
-    resetModule dirtyIdx
-    if dirtyIdx != gProjectMainIdx:
-      resetModule gProjectMainIdx
-    compileProject(cache, dirtyIdx)
+    #resetModule dirtyIdx
+    #if dirtyIdx != gProjectMainIdx:
+    #  resetModule gProjectMainIdx
+    graph.markDirty dirtyIdx
+    graph.markClientsDirty dirtyIdx
+    graph.compileProject(cache, dirtyIdx)
  if gIdeCmd in {ideUse, ideDus}:
-    let u = if suggestVersion >= 2: symFromInfo(gTrackPos) else: usageSym
+    let u = if suggestVersion >= 2: graph.symFromInfo(gTrackPos) else: usageSym
    if u != nil:
      listUsages(u)
    else:
      localError(gTrackPos, "found no symbol at this position " & $gTrackPos)

-proc executeEpc(cmd: IdeCmd, args: SexpNode; cache: IdentCache) =
+proc executeEpc(cmd: IdeCmd, args: SexpNode;
+                graph: ModuleGraph; cache: IdentCache) =
  let
    file = args[0].getStr
    line = args[1].getNum
@@ -176,7 +179,7 @@ proc executeEpc(cmd: IdeCmd, args: SexpNode; cache: IdentCache) =
  var dirtyfile = ""
  if len(args) > 3:
    dirtyfile = args[3].getStr(nil)
-  execute(cmd, file, dirtyfile, int(line), int(column), cache)
+  execute(cmd, file, dirtyfile, int(line), int(column), graph, cache)

 proc returnEpc(socket: var Socket, uid: BiggestInt, s: SexpNode|string,
               return_symbol = "return") =
@@ -192,7 +195,7 @@ template sendEpc(results: typed, tdef, hook: untyped) =
      else: s
    )

-  executeEpc(gIdeCmd, args, cache)
+  executeEpc(gIdeCmd, args, graph, cache)
  let res = sexp(results)
  if gLogging:
    logStr($res)
@@ -215,7 +218,7 @@ proc connectToNextFreePort(server: Socket, host: string): Port =
  let (_, port) = server.getLocalAddr
  result = port

-proc parseCmdLine(cmd: string; cache: IdentCache) =
+proc parseCmdLine(cmd: string; graph: ModuleGraph; cache: IdentCache) =
  template toggle(sw) =
    if sw in gGlobalOptions:
      excl(gGlobalOptions, sw)
@@ -256,25 +259,25 @@ proc parseCmdLine(cmd: string; cache: IdentCache) =
  i += skipWhile(cmd, seps, i)
  i += parseInt(cmd, col, i)

-  execute(gIdeCmd, orig, dirtyfile, line, col-1, cache)
+  execute(gIdeCmd, orig, dirtyfile, line, col-1, graph, cache)

-proc serveStdin(cache: IdentCache) =
+proc serveStdin(graph: ModuleGraph; cache: IdentCache) =
  if gEmitEof:
    echo DummyEof
    while true:
      let line = readLine(stdin)
-      parseCmdLine line, cache
+      parseCmdLine line, graph, cache
      echo DummyEof
      flushFile(stdout)
  else:
    echo Help
    var line = ""
    while readLineFromStdin("> ", line):
-      parseCmdLine line, cache
+      parseCmdLine line, graph, cache
      echo ""
      flushFile(stdout)

-proc serveTcp(cache: IdentCache) =
+proc serveTcp(graph: ModuleGraph; cache: IdentCache) =
  var server = newSocket()
  server.bindAddr(gPort, gAddress)
  var inp = "".TaintedString
@@ -288,12 +291,12 @@ proc serveTcp(cache: IdentCache) =
    accept(server, stdoutSocket)

    stdoutSocket.readLine(inp)
-    parseCmdLine inp.string, cache
+    parseCmdLine inp.string, graph, cache

    stdoutSocket.send("\c\L")
    stdoutSocket.close()

-proc serveEpc(server: Socket; cache: IdentCache) =
+proc serveEpc(server: Socket; graph: ModuleGraph; cache: IdentCache) =
  var client = newSocket()
  # Wait for connection
  accept(server, client)
@@ -346,11 +349,7 @@ proc serveEpc(server: Socket; cache: IdentCache) =
                         "unexpected call: " & epcAPI
      raise newException(EUnexpectedCommand, errMessage)

-template beCompatible() =
-  when compiles(modules.gFuzzyGraphChecking):
-    modules.gFuzzyGraphChecking = true
-
-proc mainCommand(cache: IdentCache) =
+proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
  clearPasses()
  registerPass verbosePass
  registerPass semPass
@@ -368,26 +367,23 @@ proc mainCommand(cache: IdentCache) =

  case gMode
  of mstdin:
-    beCompatible()
-    compileProject(cache)
+    compileProject(graph, cache)
    #modules.gFuzzyGraphChecking = false
-    serveStdin(cache)
+    serveStdin(graph, cache)
  of mtcp:
    # until somebody accepted the connection, produce no output (logging is too
    # slow for big projects):
    msgs.writelnHook = proc (msg: string) = discard
-    beCompatible()
-    compileProject(cache)
+    compileProject(graph, cache)
    #modules.gFuzzyGraphChecking = false
-    serveTcp(cache)
+    serveTcp(graph, cache)
  of mepc:
-    beCompatible()
    var server = newSocket()
    let port = connectToNextFreePort(server, "localhost")
    server.listen()
    echo port
-    compileProject(cache)
-    serveEpc(server, cache)
+    compileProject(graph, cache)
+    serveEpc(server, graph, cache)

 proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
  var p = parseopt.initOptParser(cmd)
@@ -464,7 +460,9 @@ proc handleCmdLine(cache: IdentCache) =
    extccomp.initVars()
    processCmdLine(passCmd2, "")

-    mainCommand(cache)
+    let graph = newModuleGraph()
+    graph.suggestMode = true
+    mainCommand(graph, cache)

 when false:
  proc quitCalled() {.noconv.} =
--- a/tools/nimsuggest/tester.nim
+++ b/tools/nimsuggest/tester.nim
@@ -27,7 +27,7 @@ proc parseTest(filename: string): Test =
  for x in lines(filename):
    let marker = x.find(cursorMarker)+1
    if marker > 0:
-      markers.add filename & ";" & dest & ":" & $i & ":" & $marker
+      markers.add "\"" & filename & "\";\"" & dest & "\":" & $i & ":" & $marker
      tmp.writeLine x.replace(cursorMarker, "")
    else:
      tmp.writeLine x
@@ -90,7 +90,7 @@ proc runTest(filename: string): int =

 proc main() =
  var failures = 0
-  for x in walkFiles("tests/t*.nim"):
+  for x in walkFiles(getAppDir() / "tests/t*.nim"):
    echo "Test ", x
    failures += runTest(expandFilename(x))
  if failures > 0: