incremental compilation: implemented basic replay logic

2026-06-04 10:54:42 +00:00 · 2018-06-02 09:41:27 +02:00
parent cae1973856
commit 826c1e2d78
25 changed files with 295 additions and 271 deletions
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -1058,22 +1058,6 @@ proc newTree*(kind: TNodeKind; children: varargs[PNode]): PNode =
    result.info = children[0].info
  result.sons = @children

-proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode =
-  result = newNode(kind)
-  result.intVal = intVal
-
-proc newIntTypeNode*(kind: TNodeKind, intVal: BiggestInt, typ: PType): PNode =
-  result = newIntNode(kind, intVal)
-  result.typ = typ
-
-proc newFloatNode*(kind: TNodeKind, floatVal: BiggestFloat): PNode =
-  result = newNode(kind)
-  result.floatVal = floatVal
-
-proc newStrNode*(kind: TNodeKind, strVal: string): PNode =
-  result = newNode(kind)
-  result.strVal = strVal
-
 template previouslyInferred*(t: PType): PType =
  if t.sons.len > 1: t.lastSon else: nil

@@ -1221,6 +1205,26 @@ proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
  result.info = info
  result.typ = typ

+proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode =
+  result = newNode(kind)
+  result.intVal = intVal
+
+proc newIntTypeNode*(kind: TNodeKind, intVal: BiggestInt, typ: PType): PNode =
+  result = newIntNode(kind, intVal)
+  result.typ = typ
+
+proc newFloatNode*(kind: TNodeKind, floatVal: BiggestFloat): PNode =
+  result = newNode(kind)
+  result.floatVal = floatVal
+
+proc newStrNode*(kind: TNodeKind, strVal: string): PNode =
+  result = newNode(kind)
+  result.strVal = strVal
+
+proc newStrNode*(strVal: string; info: TLineInfo): PNode =
+  result = newNodeI(nkStrLit, info)
+  result.strVal = strVal
+
 proc addSon*(father, son: PNode) =
  assert son != nil
  if isNil(father.sons): father.sons = @[]
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -1316,7 +1316,7 @@ template injectG() {.dirty.} =
    graph.backend = newModuleList(graph)
  let g = BModuleList(graph.backend)

-proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
  injectG()
  result = newModule(g, module, graph.config)
  if optGenIndex in graph.config.globalOptions and g.generatedHeader == nil:
@@ -1360,11 +1360,12 @@ proc getCFile(m: BModule): string =
      else: ".c"
  result = changeFileExt(completeCFilePath(m.config, withPackageName(m.config, m.cfilename)), ext)

-proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
-  injectG()
-  var m = newModule(g, module, graph.config)
-  readMergeInfo(getCFile(m), m)
-  result = m
+when false:
+  proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
+    injectG()
+    var m = newModule(g, module, graph.config)
+    readMergeInfo(getCFile(m), m)
+    result = m

 proc myProcess(b: PPassContext, n: PNode): PNode =
  result = n
@@ -1506,4 +1507,4 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
  writeMapping(config, g.mapping)
  if g.generatedHeader != nil: writeHeader(g.generatedHeader)

-const cgenPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
+const cgenPass* = makePass(myOpen, myProcess, myClose)
--- a/compiler/depends.nim
+++ b/compiler/depends.nim
@@ -51,7 +51,7 @@ proc generateDot*(graph: ModuleGraph; project: string) =
      rope(changeFileExt(extractFilename(project), "")), b.dotGraph],
            changeFileExt(project, "dot"))

-proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
  var g: PGen
  new(g)
  g.module = module
--- a/compiler/docgen2.nim
+++ b/compiler/docgen2.nim
@@ -51,7 +51,7 @@ proc processNodeJson(c: PPassContext, n: PNode): PNode =
  var g = PGen(c)
  generateJson(g.doc, n)

-proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
  var g: PGen
  new(g)
  g.module = module
--- a/compiler/importer.nim
+++ b/compiler/importer.nim
@@ -133,7 +133,7 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
    result = createModuleAlias(realModule, n.sons[1].ident, realModule.info,
                               c.config.options)

-proc myImportModule(c: PContext, n: PNode): PSym =
+proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
  var f = checkModuleName(c.config, n)
  if f != InvalidFileIDX:
    let L = c.graph.importStack.len
@@ -147,7 +147,7 @@ proc myImportModule(c: PContext, n: PNode): PSym =
        err.add toFullPath(c.config, c.graph.importStack[i]) & " imports " &
                toFullPath(c.config, c.graph.importStack[i+1])
      c.recursiveDep = err
-    result = importModuleAs(c, n, gImportModule(c.graph, c.module, f, c.cache))
+    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
    #echo "set back to ", L
    c.graph.importStack.setLen(L)
    # we cannot perform this check reliably because of
@@ -162,9 +162,10 @@ proc myImportModule(c: PContext, n: PNode): PSym =
      else:
        message(c.config, n.info, warnDeprecated, result.name.s)
    suggestSym(c.config, n.info, result, c.graph.usageSym, false)
+    importStmtResult.add newStrNode(toFullPath(c.config, f), n.info)

-proc impMod(c: PContext; it: PNode) =
-  let m = myImportModule(c, it)
+proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
+  let m = myImportModule(c, it, importStmtResult)
  if m != nil:
    var emptySet: IntSet
    # ``addDecl`` needs to be done before ``importAllSymbols``!
@@ -173,7 +174,8 @@ proc impMod(c: PContext; it: PNode) =
    #importForwarded(c, m.ast, emptySet)

 proc evalImport(c: PContext, n: PNode): PNode =
-  result = n
+  #result = n
+  result = newNodeI(nkImportStmt, n.info)
  for i in countup(0, sonsLen(n) - 1):
    let it = n.sons[i]
    if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
@@ -185,14 +187,14 @@ proc evalImport(c: PContext, n: PNode): PNode =
      a.add sep # dummy entry, replaced in the loop
      for x in it[2]:
        a.sons[2] = x
-        impMod(c, a)
+        impMod(c, a, result)
    else:
-      impMod(c, it)
+      impMod(c, it, result)

 proc evalFrom(c: PContext, n: PNode): PNode =
-  result = n
+  result = newNodeI(nkImportStmt, n.info)
  checkMinSonsLen(n, 2, c.config)
-  var m = myImportModule(c, n.sons[0])
+  var m = myImportModule(c, n.sons[0], result)
  if m != nil:
    n.sons[0] = newSymNode(m)
    addDecl(c, m, n.info)               # add symbol to symbol table of module
@@ -201,9 +203,9 @@ proc evalFrom(c: PContext, n: PNode): PNode =
        importSymbol(c, n.sons[i], m)

 proc evalImportExcept*(c: PContext, n: PNode): PNode =
-  result = n
+  result = newNodeI(nkImportStmt, n.info)
  checkMinSonsLen(n, 2, c.config)
-  var m = myImportModule(c, n.sons[0])
+  var m = myImportModule(c, n.sons[0], result)
  if m != nil:
    n.sons[0] = newSymNode(m)
    addDecl(c, m, n.info)               # add symbol to symbol table of module
--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -2277,12 +2277,8 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
    for obj, content in items(globals.classes):
      genClass(m.config, obj, content, ext)

-proc myOpenCached(graph: ModuleGraph; s: PSym, rd: PRodReader): PPassContext =
-  internalError(graph.config, "symbol files are not possible with the JS code generator")
-  result = nil
-
-proc myOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; s: PSym): PPassContext =
  result = newModule(graph, s)

-const JSgenPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
+const JSgenPass* = makePass(myOpen, myProcess, myClose)

--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -40,37 +40,37 @@ proc writeDepsFile(g: ModuleGraph; project: string) =
      f.writeLine(toFullPath(g.config, k))
  f.close()

-proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) =
+proc commandGenDepend(graph: ModuleGraph) =
  semanticPasses(graph)
  registerPass(graph, gendependPass)
-  compileProject(graph, cache)
+  compileProject(graph)
  let project = graph.config.projectFull
  writeDepsFile(graph, project)
  generateDot(graph, project)
  execExternalProgram(graph.config, "dot -Tpng -o" & changeFileExt(project, "png") &
      ' ' & changeFileExt(project, "dot"))

-proc commandCheck(graph: ModuleGraph; cache: IdentCache) =
+proc commandCheck(graph: ModuleGraph) =
  graph.config.errorMax = high(int)  # do not stop after first error
  defineSymbol(graph.config.symbols, "nimcheck")
  semanticPasses(graph)  # use an empty backend for semantic checking only
-  compileProject(graph, cache)
+  compileProject(graph)

-proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) =
+proc commandDoc2(graph: ModuleGraph; json: bool) =
  graph.config.errorMax = high(int)  # do not stop after first error
  semanticPasses(graph)
  if json: registerPass(graph, docgen2JsonPass)
  else: registerPass(graph, docgen2Pass)
-  compileProject(graph, cache)
+  compileProject(graph)
  finishDoc2Pass(graph.config.projectName)

-proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) =
+proc commandCompileToC(graph: ModuleGraph) =
  let conf = graph.config
  extccomp.initVars(conf)
  semanticPasses(graph)
  registerPass(graph, cgenPass)

-  compileProject(graph, cache)
+  compileProject(graph)
  cgenWriteModules(graph.backend, conf)
  if conf.cmd != cmdRun:
    let proj = changeFileExt(conf.projectFull, "")
@@ -79,11 +79,11 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) =
    if optGenScript in graph.config.globalOptions:
      writeDepsFile(graph, toGeneratedFile(conf, proj, ""))

-proc commandJsonScript(graph: ModuleGraph; cache: IdentCache) =
+proc commandJsonScript(graph: ModuleGraph) =
  let proj = changeFileExt(graph.config.projectFull, "")
  extccomp.runJsonBuildInstructions(graph.config, proj)

-proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) =
+proc commandCompileToJS(graph: ModuleGraph) =
  #incl(gGlobalOptions, optSafeCode)
  setTarget(graph.config.target, osJS, cpuJS)
  #initDefines()
@@ -91,9 +91,9 @@ proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) =
  defineSymbol(graph.config.symbols, "js")
  semanticPasses(graph)
  registerPass(graph, JSgenPass)
-  compileProject(graph, cache)
+  compileProject(graph)

-proc interactivePasses(graph: ModuleGraph; cache: IdentCache) =
+proc interactivePasses(graph: ModuleGraph) =
  initDefines(graph.config.symbols)
  defineSymbol(graph.config.symbols, "nimscript")
  when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
@@ -101,29 +101,29 @@ proc interactivePasses(graph: ModuleGraph; cache: IdentCache) =
  registerPass(graph, semPass)
  registerPass(graph, evalPass)

-proc commandInteractive(graph: ModuleGraph; cache: IdentCache) =
+proc commandInteractive(graph: ModuleGraph) =
  graph.config.errorMax = high(int)  # do not stop after first error
-  interactivePasses(graph, cache)
-  compileSystemModule(graph, cache)
+  interactivePasses(graph)
+  compileSystemModule(graph)
  if graph.config.commandArgs.len > 0:
-    discard graph.compileModule(fileInfoIdx(graph.config, graph.config.projectFull), cache, {})
+    discard graph.compileModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
  else:
    var m = graph.makeStdinModule()
    incl(m.flags, sfMainModule)
-    processModule(graph, m, llStreamOpenStdIn(), nil, cache)
+    processModule(graph, m, llStreamOpenStdIn())

 const evalPasses = [verbosePass, semPass, evalPass]

-proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym; cache: IdentCache) =
-  carryPasses(graph, nodes, module, cache, evalPasses)
+proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym) =
+  carryPasses(graph, nodes, module, evalPasses)

-proc commandEval(graph: ModuleGraph; cache: IdentCache; exp: string) =
+proc commandEval(graph: ModuleGraph; exp: string) =
  if graph.systemModule == nil:
-    interactivePasses(graph, cache)
-    compileSystemModule(graph, cache)
+    interactivePasses(graph)
+    compileSystemModule(graph)
  let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
-  evalNim(graph, echoExp.parseString(cache, graph.config),
-    makeStdinModule(graph), cache)
+  evalNim(graph, echoExp.parseString(graph.cache, graph.config),
+    makeStdinModule(graph))

 proc commandScan(cache: IdentCache, config: ConfigRef) =
  var f = addFileExt(mainCommandArg(config), NimExt)
@@ -145,8 +145,9 @@ proc commandScan(cache: IdentCache, config: ConfigRef) =
 const
  PrintRopeCacheStats = false

-proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
+proc mainCommand*(graph: ModuleGraph) =
  let conf = graph.config
+  let cache = graph.cache

  setupModuleCache(graph)
  # In "nim serve" scenario, each command must reset the registered passes
@@ -158,25 +159,25 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
  of "c", "cc", "compile", "compiletoc":
    # compile means compileToC currently
    conf.cmd = cmdCompileToC
-    commandCompileToC(graph, cache)
+    commandCompileToC(graph)
  of "cpp", "compiletocpp":
    conf.cmd = cmdCompileToCpp
    defineSymbol(graph.config.symbols, "cpp")
-    commandCompileToC(graph, cache)
+    commandCompileToC(graph)
  of "objc", "compiletooc":
    conf.cmd = cmdCompileToOC
    defineSymbol(graph.config.symbols, "objc")
-    commandCompileToC(graph, cache)
+    commandCompileToC(graph)
  of "run":
    conf.cmd = cmdRun
    when hasTinyCBackend:
      extccomp.setCC("tcc")
-      commandCompileToC(graph, cache)
+      commandCompileToC(graph)
    else:
      rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
  of "js", "compiletojs":
    conf.cmd = cmdCompileToJS
-    commandCompileToJS(graph, cache)
+    commandCompileToJS(graph)
  of "doc0":
    wantMainModule(conf)
    conf.cmd = cmdDoc
@@ -186,7 +187,7 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
    conf.cmd = cmdDoc
    loadConfigs(DocConfig, cache, conf)
    defineSymbol(conf.symbols, "nimdoc")
-    commandDoc2(graph, cache, false)
+    commandDoc2(graph, false)
  of "rst2html":
    conf.cmd = cmdRst2html
    loadConfigs(DocConfig, cache, conf)
@@ -207,7 +208,7 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
    loadConfigs(DocConfig, cache, conf)
    wantMainModule(conf)
    defineSymbol(conf.symbols, "nimdoc")
-    commandDoc2(graph, cache, true)
+    commandDoc2(graph, true)
  of "ctags":
    wantMainModule(conf)
    conf.cmd = cmdDoc
@@ -220,7 +221,7 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
    commandBuildIndex(cache, conf)
  of "gendepend":
    conf.cmd = cmdGenDepend
-    commandGenDepend(graph, cache)
+    commandGenDepend(graph)
  of "dump":
    conf.cmd = cmdDump
    if getConfigVar(conf, "dump.format") == "json":
@@ -249,7 +250,7 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
      for it in conf.searchPaths: msgWriteln(conf, it)
  of "check":
    conf.cmd = cmdCheck
-    commandCheck(graph, cache)
+    commandCheck(graph)
  of "parse":
    conf.cmd = cmdParse
    wantMainModule(conf)
@@ -261,15 +262,15 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
    msgWriteln(conf, "Beware: Indentation tokens depend on the parser's state!")
  of "secret":
    conf.cmd = cmdInteractive
-    commandInteractive(graph, cache)
+    commandInteractive(graph)
  of "e":
-    commandEval(graph, cache, mainCommandArg(conf))
+    commandEval(graph, mainCommandArg(conf))
  of "nop", "help":
    # prevent the "success" message:
    conf.cmd = cmdDump
  of "jsonscript":
    conf.cmd = cmdJsonScript
-    commandJsonScript(graph, cache)
+    commandJsonScript(graph)
  else:
    rawMessage(conf, errGenerated, "invalid command: " & conf.command)

--- a/compiler/modulegraphs.nim
+++ b/compiler/modulegraphs.nim
@@ -56,6 +56,9 @@ type
    opContains*, opNot*: PSym
    emptyNode*: PNode
    incr*: IncrementalCtx
+    importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
+    includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
+    recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}

 proc hash*(x: FileIndex): Hash {.borrow.}

@@ -85,6 +88,8 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
  result.opContains = createMagic(result, "contains", mInSet)
  result.emptyNode = newNode(nkEmpty)
  init(result.incr)
+  result.recordStmt = proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.} =
+    discard

 proc resetAllModules*(g: ModuleGraph) =
  initStrTable(packageSyms)
--- a/compiler/modules.nim
+++ b/compiler/modules.nim
@@ -58,52 +58,30 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
  # strTableIncl() for error corrections:
  discard strTableIncl(packSym.tab, result)

-proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; cache: IdentCache, flags: TSymFlags): PSym =
+proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
  result = graph.getModule(fileIdx)
  if result == nil:
-    #growCache gMemCacheData, fileIdx
-    #gMemCacheData[fileIdx].needsRecompile = Probing
    result = newModule(graph, fileIdx)
-    var rd: PRodReader
    result.flags = result.flags + flags
    if sfMainModule in result.flags:
      graph.config.mainPackageId = result.owner.id

-    when false:
-      if conf.cmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
-        rd = handleSymbolFile(result, cache)
-        if result.id < 0:
-          internalError("handleSymbolFile should have set the module's ID")
-          return
-      else:
-        discard
    result.id = getModuleId(graph, fileIdx, toFullPath(graph.config, fileIdx))
    discard processModule(graph, result,
-      if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil,
-      rd, cache)
-    #if optCaasEnabled in gGlobalOptions:
-    #  gMemCacheData[fileIdx].needsRecompile = Recompiled
-    #  if validFile: doHash fileIdx
+      if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil)
  elif graph.isDirty(result):
    result.flags.excl sfDirty
    # reset module fields:
    initStrTable(result.tab)
    result.ast = nil
    discard processModule(graph, result,
-      if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil,
-      nil, cache)
+      if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil)
    graph.markClientsDirty(fileIdx)
-    when false:
-      if checkDepMem(fileIdx) == Yes:
-        result = compileModule(fileIdx, cache, flags)
-      else:
-        result = gCompiledModules[fileIdx]

-proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex;
-                   cache: IdentCache): PSym {.procvar.} =
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.procvar.} =
  # this is called by the semantic checking phase
  assert graph.config != nil
-  result = compileModule(graph, fileIdx, cache, {})
+  result = compileModule(graph, fileIdx, {})
  graph.addDep(s, fileIdx)
  #if sfSystemModule in result.flags:
  #  localError(result.info, errAttemptToRedefine, result.name.s)
@@ -112,37 +90,37 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex;
    if s.owner.id == graph.config.mainPackageId: graph.config.mainPackageNotes
    else: graph.config.foreignPackageNotes

-proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex;
-                    cache: IdentCache): PNode {.procvar.} =
-  result = syntaxes.parseFile(fileIdx, cache, graph.config)
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode {.procvar.} =
+  result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
  graph.addDep(s, fileIdx)
  graph.addIncludeDep(s.position.FileIndex, fileIdx)

-proc compileSystemModule*(graph: ModuleGraph; cache: IdentCache) =
+proc compileSystemModule*(graph: ModuleGraph) =
  if graph.systemModule == nil:
    graph.config.m.systemFileIdx = fileInfoIdx(graph.config, graph.config.libpath / "system.nim")
-    discard graph.compileModule(graph.config.m.systemFileIdx, cache, {sfSystemModule})
+    discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule})

 proc wantMainModule*(conf: ConfigRef) =
  if conf.projectFull.len == 0:
    fatal(conf, newLineInfo(conf, "command line", 1, 1), errGenerated, "command expects a filename")
  conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))

-passes.gIncludeFile = includeModule
-passes.gImportModule = importModule
+proc connectCallbacks*(graph: ModuleGraph) =
+  graph.includeFileCallback = includeModule
+  graph.importModuleCallback = importModule

-proc compileProject*(graph: ModuleGraph; cache: IdentCache;
-                     projectFileIdx = InvalidFileIDX) =
+proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIDX) =
+  connectCallbacks(graph)
  let conf = graph.config
  wantMainModule(conf)
  let systemFileIdx = fileInfoIdx(conf, conf.libpath / "system.nim")
  let projectFile = if projectFileIdx == InvalidFileIDX: conf.projectMainIdx else: projectFileIdx
  graph.importStack.add projectFile
  if projectFile == systemFileIdx:
-    discard graph.compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
+    discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
  else:
-    graph.compileSystemModule(cache)
-    discard graph.compileModule(projectFile, cache, {sfMainModule})
+    graph.compileSystemModule()
+    discard graph.compileModule(projectFile, {sfMainModule})

 proc makeModule*(graph: ModuleGraph; filename: string): PSym =
  result = graph.newModule(fileInfoIdx(graph.config, filename))
--- a/compiler/nim.nim
+++ b/compiler/nim.nim
@@ -94,7 +94,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
    processCmdLine(passCmd2, "", conf)
    if conf.command == "":
      rawMessage(conf, errGenerated, "command missing")
-    mainCommand(newModuleGraph(cache, conf), cache)
+    mainCommand(newModuleGraph(cache, conf))
    if optHints in conf.options and hintGCStats in conf.notes: echo(GC_getStatistics())
    #echo(GC_getStatistics())
    if conf.errorCounter == 0:
--- a/compiler/nimeval.nim
+++ b/compiler/nimeval.nim
@@ -95,6 +95,7 @@ proc createInterpreter*(scriptName: string;
  var conf = newConfigRef()
  var cache = newIdentCache()
  var graph = newModuleGraph(cache, conf)
+  connectCallbacks(graph)
  initDefines(conf.symbols)
  defineSymbol(conf.symbols, "nimscript")
  defineSymbol(conf.symbols, "nimconfig")
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -238,6 +238,19 @@ type
    structuredErrorHook*: proc (config: ConfigRef; info: TLineInfo; msg: string;
                                severity: Severity) {.closure.}

+template depConfigFields*(fn) {.dirty.} =
+  fn(target)
+  fn(options)
+  fn(globalOptions)
+  fn(selectedGC)
+
+template serializeConfigFields(fn) {.dirty.} =
+  fn(cppDefines)
+  fn(externalToLink)
+  fn(linkOptions)
+  fn(compileOptions)
+  fn(toCompile)
+
 const oldExperimentalFeatures* = {implicitDeref, dotOperators, callOperator, parallel}

 const
--- a/compiler/passaux.nim
+++ b/compiler/passaux.nim
@@ -18,7 +18,7 @@ type
  VerboseRef = ref object of TPassContext
    config: ConfigRef

-proc verboseOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext =
+proc verboseOpen(graph: ModuleGraph; s: PSym): PPassContext =
  #MessageOut('compiling ' + s.name.s);
  result = VerboseRef(config: graph.config)
  rawMessage(graph.config, hintProcessing, s.name.s)
--- a/compiler/passes.nim
+++ b/compiler/passes.nim
@@ -18,19 +18,15 @@ import


 type
-  PRodReader* = ref object
  TPassContext* = object of RootObj # the pass's context

  PPassContext* = ref TPassContext

-  TPassOpen* = proc (graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext {.nimcall.}
-  TPassOpenCached* =
-    proc (graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext {.nimcall.}
+  TPassOpen* = proc (graph: ModuleGraph; module: PSym): PPassContext {.nimcall.}
  TPassClose* = proc (graph: ModuleGraph; p: PPassContext, n: PNode): PNode {.nimcall.}
  TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}

-  TPass* = tuple[open: TPassOpen, openCached: TPassOpenCached,
-                 process: TPassProcess, close: TPassClose,
+  TPass* = tuple[open: TPassOpen, process: TPassProcess, close: TPassClose,
                 isFrontend: bool]

  TPassData* = tuple[input: PNode, closeOutput: PNode]
@@ -41,23 +37,14 @@ type
 # This mechanism used to be used for the instantiation of generics.

 proc makePass*(open: TPassOpen = nil,
-               openCached: TPassOpenCached = nil,
               process: TPassProcess = nil,
               close: TPassClose = nil,
               isFrontend = false): TPass =
  result.open = open
-  result.openCached = openCached
  result.close = close
  result.process = process
  result.isFrontend = isFrontend

-# the semantic checker needs these:
-var
-  gImportModule*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex; cache: IdentCache): PSym {.nimcall.}
-  gIncludeFile*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex; cache: IdentCache): PNode {.nimcall.}
-
-# implementation
-
 proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} =
  # can be used by codegen passes to determine whether they should do
  # something with `n`. Currently, this ignores `n` and uses the global
@@ -81,35 +68,27 @@ proc registerPass*(g: ModuleGraph; p: TPass) =
  gPasses[gPassesLen] = p
  inc(gPassesLen)

-proc carryPass*(g: ModuleGraph; p: TPass, module: PSym; cache: IdentCache;
+proc carryPass*(g: ModuleGraph; p: TPass, module: PSym;
                m: TPassData): TPassData =
-  var c = p.open(g, module, cache)
+  var c = p.open(g, module)
  result.input = p.process(c, m.input)
  result.closeOutput = if p.close != nil: p.close(g, c, m.closeOutput)
                       else: m.closeOutput

 proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym;
-                  cache: IdentCache; passes: TPasses) =
+                  passes: TPasses) =
  var passdata: TPassData
  passdata.input = nodes
  for pass in passes:
-    passdata = carryPass(g, pass, module, cache, passdata)
+    passdata = carryPass(g, pass, module, passdata)

 proc openPasses(g: ModuleGraph; a: var TPassContextArray;
-                module: PSym; cache: IdentCache) =
+                module: PSym) =
  for i in countup(0, gPassesLen - 1):
    if not isNil(gPasses[i].open):
-      a[i] = gPasses[i].open(g, module, cache)
+      a[i] = gPasses[i].open(g, module)
    else: a[i] = nil

-proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym,
-                      rd: PRodReader) =
-  for i in countup(0, gPassesLen - 1):
-    if not isNil(gPasses[i].openCached):
-      a[i] = gPasses[i].openCached(g, module, rd)
-    else:
-      a[i] = nil
-
 proc closePasses(graph: ModuleGraph; a: var TPassContextArray) =
  var m: PNode = nil
  for i in countup(0, gPassesLen - 1):
@@ -125,19 +104,6 @@ proc processTopLevelStmt(n: PNode, a: var TPassContextArray): bool =
      if isNil(m): return false
  result = true

-proc processTopLevelStmtCached(n: PNode, a: var TPassContextArray) =
-  # this implements the code transformation pipeline
-  var m = n
-  for i in countup(0, gPassesLen - 1):
-    if not isNil(gPasses[i].openCached): m = gPasses[i].process(a[i], m)
-
-proc closePassesCached(graph: ModuleGraph; a: var TPassContextArray) =
-  var m: PNode = nil
-  for i in countup(0, gPassesLen - 1):
-    if not isNil(gPasses[i].openCached) and not isNil(gPasses[i].close):
-      m = gPasses[i].close(graph, a[i], m)
-    a[i] = nil                # free the memory here
-
 proc resolveMod(conf: ConfigRef; module, relativeTo: string): FileIndex =
  let fullPath = findModule(conf, module, relativeTo)
  if fullPath.len == 0:
@@ -159,8 +125,7 @@ proc processImplicits(conf: ConfigRef; implicits: seq[string], nodeKind: TNodeKi
      importStmt.addSon str
      if not processTopLevelStmt(importStmt, a): break

-proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
-                    rd: PRodReader; cache: IdentCache): bool {.discardable.} =
+proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {.discardable.} =
  if graph.stopCompile(): return true
  var
    p: TParsers
@@ -171,24 +136,17 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
    # new module caching mechanism:
    for i in 0..<gPassesLen:
      if not isNil(gPasses[i].open) and not gPasses[i].isFrontend:
-        a[i] = gPasses[i].open(graph, module, cache)
+        a[i] = gPasses[i].open(graph, module)
      else:
        a[i] = nil

-    var stmtIndex = 0
-    var doContinue = true
-    while doContinue:
-      let n = loadNode(graph, module, stmtIndex)
-      if n == nil or graph.stopCompile(): break
-      #if n.kind == nkImportStmt:
-      #  echo "yes and it's ", n
-      inc stmtIndex
+    if not graph.stopCompile():
+      let n = loadNode(graph, module)
      var m = n
      for i in 0..<gPassesLen:
        if not isNil(gPasses[i].process) and not gPasses[i].isFrontend:
          m = gPasses[i].process(a[i], m)
          if isNil(m):
-            doContinue = false
            break

    var m: PNode = nil
@@ -197,7 +155,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
        m = gPasses[i].close(graph, a[i], m)
      a[i] = nil
  else:
-    openPasses(graph, a, module, cache)
+    openPasses(graph, a, module)
    if stream == nil:
      let filename = toFullPathConsiderDirty(graph.config, fileIdx)
      s = llStreamOpen(filename, fmRead)
@@ -207,7 +165,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
    else:
      s = stream
    while true:
-      openParsers(p, fileIdx, s, cache, graph.config)
+      openParsers(p, fileIdx, s, graph.cache, graph.config)

      if sfSystemModule notin module.flags:
        # XXX what about caching? no processing then? what if I change the
@@ -230,7 +188,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
            if n.kind == nkEmpty: break
            sl.add n
          if sfReorder in module.flags:
-            sl = reorder(graph, sl, module, cache)
+            sl = reorder(graph, sl, module)
          discard processTopLevelStmt(sl, a)
          break
        elif not processTopLevelStmt(n, a): break
--- a/compiler/pragmas.nim
+++ b/compiler/pragmas.nim
@@ -84,6 +84,13 @@ proc getPragmaVal*(procAst: PNode; name: TSpecialWord): PNode =
 proc pragma*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords)
 # implementation

+proc recordPragma(c: PContext; n: PNode; key, val: string; val2 = "") =
+  var recorded = newNodeI(nkCommentStmt, n.info)
+  recorded.add newStrNode(key, n.info)
+  recorded.add newStrNode(val, n.info)
+  if val2.len > 0: recorded.add newStrNode(val2, n.info)
+  c.graph.recordStmt(c.graph, c.module, recorded)
+
 const
  errStringLiteralExpected = "string literal expected"
  errIntLiteralExpected = "integer literal expected"
@@ -227,7 +234,7 @@ proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) =

 proc processCallConv(c: PContext, n: PNode) =
  if n.kind in nkPragmaCallKinds and n.len == 2 and n.sons[1].kind == nkIdent:
-    var sw = whichKeyword(n.sons[1].ident)
+    let sw = whichKeyword(n.sons[1].ident)
    case sw
    of FirstCallConv..LastCallConv:
      c.optionStack[^1].defaultCC = wordToCallConv(sw)
@@ -412,6 +419,10 @@ proc relativeFile(c: PContext; n: PNode; ext=""): string =
      if result.len == 0: result = s

 proc processCompile(c: PContext, n: PNode) =
+  proc docompile(c: PContext; it: PNode; src, dest: string) =
+    var cf = Cfile(cname: src, obj: dest, flags: {CfileFlag.External})
+    extccomp.addExternalFileToCompile(c.config, cf)
+    recordPragma(c, it, "compile", src, dest)

  proc getStrLit(c: PContext, n: PNode; i: int): string =
    n.sons[i] = c.semConstExpr(c, n[i])
@@ -428,11 +439,8 @@ proc processCompile(c: PContext, n: PNode) =
    let dest = getStrLit(c, it, 1)
    var found = parentDir(toFullPath(c.config, n.info)) / s
    for f in os.walkFiles(found):
-      let nameOnly = extractFilename(f)
-      var cf = Cfile(cname: f,
-          obj: completeCFilePath(c.config, dest % nameOnly),
-          flags: {CfileFlag.External})
-      extccomp.addExternalFileToCompile(c.config, cf)
+      let obj = completeCFilePath(c.config, dest % extractFilename(f))
+      docompile(c, it, f, obj)
  else:
    let s = expectStrLit(c, n)
    var found = parentDir(toFullPath(c.config, n.info)) / s
@@ -441,15 +449,19 @@ proc processCompile(c: PContext, n: PNode) =
      else:
        found = findFile(c.config, s)
        if found.len == 0: found = s
-    extccomp.addExternalFileToCompile(c.config, found)
+    let obj = toObjFile(c.config, completeCFilePath(c.config, changeFileExt(found, ""), false))
+    docompile(c, it, found, obj)

 proc processCommonLink(c: PContext, n: PNode, feature: TLinkFeature) =
  let found = relativeFile(c, n, CC[c.config.cCompiler].objExt)
  case feature
-  of linkNormal: extccomp.addExternalFileToLink(c.config, found)
+  of linkNormal:
+    extccomp.addExternalFileToLink(c.config, found)
+    recordPragma(c, n, "link", found)
  of linkSys:
-    extccomp.addExternalFileToLink(c.config,
-      c.config.libpath / completeCFilePath(c.config, found, false))
+    let dest = c.config.libpath / completeCFilePath(c.config, found, false)
+    extccomp.addExternalFileToLink(c.config, dest)
+    recordPragma(c, n, "link", dest)
  else: internalError(c.config, n.info, "processCommonLink")

 proc pragmaBreakpoint(c: PContext, n: PNode) =
@@ -724,7 +736,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
    i.inc(userPragma.ast.len - 1) # inc by -1 is ok, user pragmas was empty
    dec c.instCounter
  else:
-    var k = whichKeyword(ident)
+    let k = whichKeyword(ident)
    if k in validPragmas:
      case k
      of wExportc:
@@ -891,8 +903,14 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
        noVal(c, it)
        if sym.typ == nil: invalidPragma(c, it)
        else: incl(sym.typ.flags, tfPacked)
-      of wHint: message(c.config, it.info, hintUser, expectStrLit(c, it))
-      of wWarning: message(c.config, it.info, warnUser, expectStrLit(c, it))
+      of wHint:
+        let s = expectStrLit(c, it)
+        recordPragma(c, it, "hint", s)
+        message(c.config, it.info, hintUser, s)
+      of wWarning:
+        let s = expectStrLit(c, it)
+        recordPragma(c, it, "warning", s)
+        message(c.config, it.info, warnUser, s)
      of wError:
        if sym != nil and sym.isRoutine:
          # This is subtle but correct: the error *statement* is only
@@ -902,7 +920,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
          noVal(c, it)
          incl(sym.flags, sfError)
        else:
-          localError(c.config, it.info, errUser, expectStrLit(c, it))
+          let s = expectStrLit(c, it)
+          recordPragma(c, it, "error", s)
+          localError(c.config, it.info, errUser, s)
      of wFatal: fatal(c.config, it.info, errUser, expectStrLit(c, it))
      of wDefine: processDefine(c, it)
      of wUndef: processUndef(c, it)
@@ -1066,8 +1086,7 @@ proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
      if sym.loc.r == nil: sym.loc.r = rope(sym.name.s)

 proc hasPragma*(n: PNode, pragma: TSpecialWord): bool =
-  if n == nil or n.sons == nil:
-    return false
+  if n == nil: return false

  for p in n:
    var key = if p.kind in nkPragmaCallKinds and p.len > 1: p[0] else: p
@@ -1079,7 +1098,7 @@ proc hasPragma*(n: PNode, pragma: TSpecialWord): bool =
 proc pragmaRec(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) =
  if n == nil: return
  var i = 0
-  while i < n.len():
+  while i < n.len:
    if singlePragma(c, sym, n, i, validPragmas): break
    inc i

--- a/compiler/reorder.nim
+++ b/compiler/reorder.nim
@@ -136,15 +136,13 @@ proc hasIncludes(n:PNode): bool =
    if a.kind == nkIncludeStmt:
      return true

-proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex;
-                    cache: IdentCache): PNode {.procvar.} =
-  result = syntaxes.parseFile(fileIdx, cache, graph.config)
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode {.procvar.} =
+  result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
  graph.addDep(s, fileIdx)
  graph.addIncludeDep(FileIndex s.position, fileIdx)

 proc expandIncludes(graph: ModuleGraph, module: PSym, n: PNode,
-                    modulePath: string, includedFiles: var IntSet,
-                    cache: IdentCache): PNode =
+                    modulePath: string, includedFiles: var IntSet): PNode =
  # Parses includes and injects them in the current tree
  if not n.hasIncludes:
    return n
@@ -158,9 +156,9 @@ proc expandIncludes(graph: ModuleGraph, module: PSym, n: PNode,
            localError(graph.config, a.info, "recursive dependency: '$1'" %
              toFilename(graph.config, f))
          else:
-            let nn = includeModule(graph, module, f, cache)
+            let nn = includeModule(graph, module, f)
            let nnn = expandIncludes(graph, module, nn, modulePath,
-                                      includedFiles, cache)
+                                      includedFiles)
            excl(includedFiles, f.int)
            for b in nnn:
              result.add b
@@ -427,19 +425,19 @@ proc hasForbiddenPragma(n: PNode): bool =
        a[0].ident.s == "push":
          return true

-proc reorder*(graph: ModuleGraph, n: PNode, module: PSym, cache: IdentCache): PNode =
+proc reorder*(graph: ModuleGraph, n: PNode, module: PSym): PNode =
  if n.hasForbiddenPragma:
    return n
  var includedFiles = initIntSet()
  let mpath = toFullPath(graph.config, module.fileIdx)
  let n = expandIncludes(graph, module, n, mpath,
-                          includedFiles, cache).splitSections
+                          includedFiles).splitSections
  result = newNodeI(nkStmtList, n.info)
  var deps = newSeq[(IntSet, IntSet)](n.len)
  for i in 0..<n.len:
    deps[i][0] = initIntSet()
    deps[i][1] = initIntSet()
-    computeDeps(cache, n[i], deps[i][0], deps[i][1], true)
+    computeDeps(graph.cache, n[i], deps[i][0], deps[i][1], true)

  var g = buildGraph(n, deps)
  let comps = getStrongComponents(g)
--- a/compiler/rod.nim
+++ b/compiler/rod.nim
@@ -14,7 +14,7 @@ import ast, idgen, lineinfos, msgs, incremental, modulegraphs
 when not nimIncremental:
  template setupModuleCache*(g: ModuleGraph) = discard
  template storeNode*(g: ModuleGraph; module: PSym; n: PNode) = discard
-  template loadNode*(g: ModuleGraph; module: PSym; index: var int): PNode = PNode(nil)
+  template loadNode*(g: ModuleGraph; module: PSym): PNode = newNode(nkStmtList)

  template getModuleId*(g: ModuleGraph; fileIdx: FileIndex; fullpath: string): int = getID()

--- a/compiler/rodimpl.nim
+++ b/compiler/rodimpl.nim
@@ -10,7 +10,7 @@
 ## This module implements the new compilation cache.

 import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
-  renderer, rodutils, idents, astalgo, btrees, magicsys
+  renderer, rodutils, idents, astalgo, btrees, magicsys, cgmeth, extccomp

 ## Todo:
 ## - Implement the 'import' replay logic so that the codegen runs over
@@ -18,7 +18,7 @@ import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
 ## - Make conditional symbols and the configuration part of a module's
 ##   dependencies.
 ## - Test multi methods.
-## - Implement the limited VM support based on sets.
+## - Implement the limited VM support based on replays.
 ## - Depencency computation should use *signature* hashes in order to
 ##   avoid recompiling dependent modules.

@@ -381,6 +381,9 @@ proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) =
      break
    inc i

+proc recordStmt*(g: ModuleGraph; module: PSym; n: PNode) =
+  storeNode(g, module, n)
+
 proc storeRemaining*(g: ModuleGraph; module: PSym) =
  if g.config.symbolFiles == disabledSf: return
  var stillForwarded: seq[PSym] = @[]
@@ -399,7 +402,6 @@ type
    pos: int

 using
-  r: var Reader
  b: var BlobReader
  g: ModuleGraph

@@ -760,21 +762,68 @@ proc loadModuleSymTab(g; module: PSym) =
  if sfSystemModule in module.flags:
    g.systemModule = module

-proc loadNode*(g: ModuleGraph; module: PSym; index: int): PNode =
-  if index == 0:
-    loadModuleSymTab(g, module)
-    #index = parseInt db.getValue(
-    #  sql"select min(id) from toplevelstmts where module = ?", abs module.id)
-  var b = BlobReader(pos: 0)
-  b.s = db.getValue(sql"select data from toplevelstmts where position = ? and module = ?",
-                    index, abs module.id)
-  if b.s.len == 0:
-    db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId)
-    return nil # end marker
-  result = decodeNode(g, b, module.info)
+proc replay(g: ModuleGraph; module: PSym; n: PNode) =
+  case n.kind
+  of nkStaticStmt:
+    #evalStaticStmt()
+    discard "XXX to implement"
+  of nkVarSection, nkLetSection:
+    #setupCompileTimeVar()
+    discard "XXX to implement"
+  of nkMethodDef:
+    methodDef(g, n[namePos].sym, fromCache=true)
+  of nkCommentStmt:
+    # pragmas are complex and can be user-overriden via templates. So
+    # instead of using the original ``nkPragma`` nodes, we rely on the
+    # fact that pragmas.nim was patched to produce specialized recorded
+    # statements for us in the form of ``nkCommentStmt`` with (key, value)
+    # pairs. Ordinary nkCommentStmt nodes never have children so this is
+    # not ambiguous.
+    # Fortunately only a tiny subset of the available pragmas need to
+    # be replayed here. This is always a subset of ``pragmas.stmtPragmas``.
+    if n.len >= 2:
+      internalAssert g.config, n[0].kind == nkStrLit and n[1].kind == nkStrLit
+      case n[0].strVal
+      of "hint": message(g.config, n.info, hintUser, n[1].strVal)
+      of "warning": message(g.config, n.info, warnUser, n[1].strVal)
+      of "error": localError(g.config, n.info, errUser, n[1].strVal)
+      of "compile":
+        internalAssert g.config, n.len == 3 and n[2].kind == nkStrLit
+        var cf = Cfile(cname: n[1].strVal, obj: n[2].strVal,
+                       flags: {CfileFlag.External})
+        extccomp.addExternalFileToCompile(g.config, cf)
+      of "link":
+        extccomp.addExternalFileToLink(g.config, n[1].strVal)
+      else:
+        internalAssert g.config, false
+  of nkImportStmt:
+    for x in n:
+      if x.kind == nkStrLit:
+        # XXX check that importModuleCallback implements the right logic
+        let imported = g.importModuleCallback(g, module, fileInfoIdx(g.config, n[0].strVal))
+        internalAssert g.config, imported.id < 0
+  of nkStmtList, nkStmtListExpr:
+    for x in n: replay(g, module, x)
+  else: discard "nothing to do for this node"
+
+proc loadNode*(g: ModuleGraph; module: PSym): PNode =
+  loadModuleSymTab(g, module)
+  result = newNodeI(nkStmtList, module.info)
+  for row in db.rows(sql"select data from toplevelstmts where module = ? order by position asc",
+                        abs module.id):
+
+    var b = BlobReader(pos: 0)
+    shallowCopy b.s, row[0]
+    # ensure we can read without index checks:
+    b.s.add '\0'
+    result.add decodeNode(g, b, module.info)
+
+  db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId)
+  replay(g, module, result)

 proc setupModuleCache*(g: ModuleGraph) =
  if g.config.symbolFiles == disabledSf: return
+  g.recordStmt = recordStmt
  let dbfile = getNimcacheDir(g.config) / "rodfiles.db"
  if not fileExists(dbfile):
    db = open(connection=dbfile, user="nim", password="",
--- a/compiler/scriptconfig.nim
+++ b/compiler/scriptconfig.nim
@@ -154,6 +154,7 @@ proc runNimScript*(cache: IdentCache; scriptName: string;
  rawMessage(conf, hintConf, scriptName)

  let graph = newModuleGraph(cache, conf)
+  connectCallbacks(graph)
  if freshDefines: initDefines(conf.symbols)

  defineSymbol(conf.symbols, "nimscript")
@@ -167,8 +168,8 @@ proc runNimScript*(cache: IdentCache; scriptName: string;
  incl(m.flags, sfMainModule)
  graph.vm = setupVM(m, cache, scriptName, graph)

-  graph.compileSystemModule(cache)
-  discard graph.processModule(m, llStreamOpen(scriptName, fmRead), nil, cache)
+  graph.compileSystemModule()
+  discard graph.processModule(m, llStreamOpen(scriptName, fmRead))

  # ensure we load 'system.nim' again for the real non-config stuff!
  resetSystemArtifacts(graph)
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -314,7 +314,7 @@ proc tryConstExpr(c: PContext, n: PNode): PNode =
  c.config.errorMax = high(int)

  try:
-    result = evalConstExpr(c.module, c.cache, c.graph, e)
+    result = evalConstExpr(c.module, c.graph, e)
    if result == nil or result.kind == nkEmpty:
      result = nil
    else:
@@ -338,7 +338,7 @@ proc semConstExpr(c: PContext, n: PNode): PNode =
  result = getConstExpr(c.module, e, c.graph)
  if result == nil:
    #if e.kind == nkEmpty: globalError(n.info, errConstExprExpected)
-    result = evalConstExpr(c.module, c.cache, c.graph, e)
+    result = evalConstExpr(c.module, c.graph, e)
    if result == nil or result.kind == nkEmpty:
      if e.info != n.info:
        pushInfoContext(c.config, n.info)
@@ -446,7 +446,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,

  #if c.evalContext == nil:
  #  c.evalContext = c.createEvalContext(emStatic)
-  result = evalMacroCall(c.module, c.cache, c.graph, n, nOrig, sym)
+  result = evalMacroCall(c.module, c.graph, n, nOrig, sym)
  if efNoSemCheck notin flags:
    result = semAfterMacroCall(c, n, result, sym, flags)
  result = wrapInComesFrom(nOrig.info, sym, result)
@@ -482,7 +482,7 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
        addSon(n, prc.ast)
  c.lastGenericIdx = c.generics.len

-proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
  var c = newContext(graph, module)
  if c.p != nil: internalError(graph.config, module.info, "sem.myOpen")
  c.semConstExpr = semConstExpr
@@ -512,9 +512,6 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
    graph.config.notes = graph.config.foreignPackageNotes
  result = c

-proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext =
-  result = myOpen(graph, module, graph.cache)
-
 proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool =
  if g.systemModule == nil: return false
  case n.kind
@@ -625,5 +622,5 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
  storeRemaining(c.graph, c.module)
  if c.runnableExamples != nil: testExamples(c)

-const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose,
+const semPass* = makePass(myOpen, myProcess, myClose,
                          isFrontend = true)
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -617,12 +617,12 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
      call.add(a)
    #echo "NOW evaluating at compile time: ", call.renderTree
    if sfCompileTime in callee.flags:
-      result = evalStaticExpr(c.module, c.cache, c.graph, call, c.p.owner)
+      result = evalStaticExpr(c.module, c.graph, call, c.p.owner)
      if result.isNil:
        localError(c.config, n.info, errCannotInterpretNodeX % renderTree(call))
      else: result = fixupTypeAfterEval(c, result, n)
    else:
-      result = evalConstExpr(c.module, c.cache, c.graph, call)
+      result = evalConstExpr(c.module, c.graph, call)
      if result.isNil: result = n
      else: result = fixupTypeAfterEval(c, result, n)
    #if result != n:
@@ -631,7 +631,7 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
 proc semStaticExpr(c: PContext, n: PNode): PNode =
  let a = semExpr(c, n.sons[0])
  if a.findUnresolvedStatic != nil: return a
-  result = evalStaticExpr(c.module, c.cache, c.graph, a, c.p.owner)
+  result = evalStaticExpr(c.module, c.graph, a, c.p.owner)
  if result.isNil:
    localError(c.config, n.info, errCannotInterpretNodeX % renderTree(n))
    result = c.graph.emptyNode
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -572,7 +572,8 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
      if v.flags * {sfGlobal, sfThread} == {sfGlobal}:
        message(c.config, v.info, hintGlobalVar)
  if hasCompileTime:
-    vm.setupCompileTimeVar(c.module, c.cache, c.graph, result)
+    vm.setupCompileTimeVar(c.module, c.graph, result)
+    c.graph.recordStmt(c.graph, c.module, result)

 proc semConst(c: PContext, n: PNode): PNode =
  result = copyNode(n)
@@ -1059,7 +1060,7 @@ proc semAllTypeSections(c: PContext; n: PNode): PNode =
          if containsOrIncl(c.includedFiles, f.int):
            localError(c.config, n.info, errRecursiveDependencyX % toFilename(c.config, f))
          else:
-            let code = gIncludeFile(c.graph, c.module, f, c.cache)
+            let code = c.graph.includeFileCallback(c.graph, c.module, f)
            gatherStmts c, code, result
            excl(c.includedFiles, f.int)
    of nkStmtList:
@@ -1741,7 +1742,7 @@ proc evalInclude(c: PContext, n: PNode): PNode =
      if containsOrIncl(c.includedFiles, f.int):
        localError(c.config, n.info, errRecursiveDependencyX % toFilename(c.config, f))
      else:
-        addSon(result, semStmt(c, gIncludeFile(c.graph, c.module, f, c.cache)))
+        addSon(result, semStmt(c, c.graph.includeFileCallback(c.graph, c.module, f)))
        excl(c.includedFiles, f.int)

 proc setLine(n: PNode, info: TLineInfo) =
@@ -1770,7 +1771,7 @@ proc semStaticStmt(c: PContext, n: PNode): PNode =
  let a = semStmt(c, n.sons[0])
  dec c.inStaticContext
  n.sons[0] = a
-  evalStaticStmt(c.module, c.cache, c.graph, a, c.p.owner)
+  evalStaticStmt(c.module, c.graph, a, c.p.owner)
  result = newNodeI(nkDiscardStmt, n.info, 1)
  result.sons[0] = c.graph.emptyNode

--- a/compiler/suggest.nim
+++ b/compiler/suggest.nim
@@ -323,7 +323,7 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
        # error: no known module name:
        typ = nil
      else:
-        let m = gImportModule(c.graph, c.module, fileInfoIdx(c.config, fullpath), c.cache)
+        let m = c.graph.importModuleCallback(c.graph, c.module, fileInfoIdx(c.config, fullpath))
        if m == nil: typ = nil
        else:
          for it in items(n.sym.tab):
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -1657,20 +1657,20 @@ proc getGlobalValue*(c: PCtx; s: PSym): PNode =

 include vmops

-proc setupGlobalCtx(module: PSym; cache: IdentCache; graph: ModuleGraph) =
+proc setupGlobalCtx(module: PSym; graph: ModuleGraph) =
  if graph.vm.isNil:
-    graph.vm = newCtx(module, cache, graph)
+    graph.vm = newCtx(module, graph.cache, graph)
    registerAdditionalOps(PCtx graph.vm)
  else:
    refresh(PCtx graph.vm, module)

-proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
  #var c = newEvalContext(module, emRepl)
  #c.features = {allowCast, allowFFI, allowInfiniteLoops}
  #pushStackFrame(c, newStackFrame())

  # XXX produce a new 'globals' environment here:
-  setupGlobalCtx(module, cache, graph)
+  setupGlobalCtx(module, graph)
  result = PCtx graph.vm
  when hasFFI:
    PCtx(graph.vm).features = {allowFFI, allowCast}
@@ -1688,13 +1688,13 @@ proc myProcess(c: PPassContext, n: PNode): PNode =
 proc myClose(graph: ModuleGraph; c: PPassContext, n: PNode): PNode =
  myProcess(c, n)

-const evalPass* = makePass(myOpen, nil, myProcess, myClose)
+const evalPass* = makePass(myOpen, myProcess, myClose)

-proc evalConstExprAux(module: PSym; cache: IdentCache;
+proc evalConstExprAux(module: PSym;
                      g: ModuleGraph; prc: PSym, n: PNode,
                      mode: TEvalMode): PNode =
  let n = transformExpr(g, module, n)
-  setupGlobalCtx(module, cache, g)
+  setupGlobalCtx(module, g)
  var c = PCtx g.vm
  let oldMode = c.mode
  defer: c.mode = oldMode
@@ -1709,17 +1709,17 @@ proc evalConstExprAux(module: PSym; cache: IdentCache;
  result = rawExecute(c, start, tos).regToNode
  if result.info.col < 0: result.info = n.info

-proc evalConstExpr*(module: PSym; cache: IdentCache, g: ModuleGraph; e: PNode): PNode =
-  result = evalConstExprAux(module, cache, g, nil, e, emConst)
+proc evalConstExpr*(module: PSym; g: ModuleGraph; e: PNode): PNode =
+  result = evalConstExprAux(module, g, nil, e, emConst)

-proc evalStaticExpr*(module: PSym; cache: IdentCache, g: ModuleGraph; e: PNode, prc: PSym): PNode =
-  result = evalConstExprAux(module, cache, g, prc, e, emStaticExpr)
+proc evalStaticExpr*(module: PSym; g: ModuleGraph; e: PNode, prc: PSym): PNode =
+  result = evalConstExprAux(module, g, prc, e, emStaticExpr)

-proc evalStaticStmt*(module: PSym; cache: IdentCache, g: ModuleGraph; e: PNode, prc: PSym) =
-  discard evalConstExprAux(module, cache, g, prc, e, emStaticStmt)
+proc evalStaticStmt*(module: PSym; g: ModuleGraph; e: PNode, prc: PSym) =
+  discard evalConstExprAux(module, g, prc, e, emStaticStmt)

-proc setupCompileTimeVar*(module: PSym; cache: IdentCache, g: ModuleGraph; n: PNode) =
-  discard evalConstExprAux(module, cache, g, nil, n, emStaticStmt)
+proc setupCompileTimeVar*(module: PSym; g: ModuleGraph; n: PNode) =
+  discard evalConstExprAux(module, g, nil, n, emStaticStmt)

 proc setupMacroParam(x: PNode, typ: PType): TFullReg =
  case typ.kind
@@ -1746,7 +1746,7 @@ iterator genericParamsInMacroCall*(macroSym: PSym, call: PNode): (PSym, PNode) =
 # to prevent endless recursion in macro instantiation
 const evalMacroLimit = 1000

-proc evalMacroCall*(module: PSym; cache: IdentCache; g: ModuleGraph;
+proc evalMacroCall*(module: PSym; g: ModuleGraph;
                    n, nOrig: PNode, sym: PSym): PNode =
  # XXX globalError() is ugly here, but I don't know a better solution for now
  inc(g.config.evalMacroCounter)
@@ -1759,7 +1759,7 @@ proc evalMacroCall*(module: PSym; cache: IdentCache; g: ModuleGraph;
    globalError(g.config, n.info, "in call '$#' got $#, but expected $# argument(s)" % [
        n.renderTree, $(n.safeLen-1), $(sym.typ.len-1)])

-  setupGlobalCtx(module, cache, g)
+  setupGlobalCtx(module, g)
  var c = PCtx g.vm
  c.comesFromHeuristic.line = 0'u16

--- a/nimsuggest/nimsuggest.nim
+++ b/nimsuggest/nimsuggest.nim
@@ -159,7 +159,7 @@ proc symFromInfo(graph: ModuleGraph; trackPos: TLineInfo): PSym =
    result = findNode(m.ast, trackPos)

 proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
-             graph: ModuleGraph; cache: IdentCache) =
+             graph: ModuleGraph) =
  let conf = graph.config
  myLog("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile &
        "[" & $line & ":" & $col & "]")
@@ -184,7 +184,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
  if conf.suggestVersion == 1:
    graph.usageSym = nil
  if not isKnownFile:
-    graph.compileProject(cache)
+    graph.compileProject()
  if conf.suggestVersion == 0 and conf.ideCmd in {ideUse, ideDus} and
      dirtyfile.len == 0:
    discard "no need to recompile anything"
@@ -193,7 +193,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
    graph.markDirty dirtyIdx
    graph.markClientsDirty dirtyIdx
    if conf.ideCmd != ideMod:
-      graph.compileProject(cache, modIdx)
+      graph.compileProject(modIdx)
  if conf.ideCmd in {ideUse, ideDus}:
    let u = if conf.suggestVersion != 1: graph.symFromInfo(conf.m.trackPos) else: graph.usageSym
    if u != nil:
@@ -202,7 +202,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
      localError(conf, conf.m.trackPos, "found no symbol at this position " & (conf $ conf.m.trackPos))

 proc executeEpc(cmd: IdeCmd, args: SexpNode;
-                graph: ModuleGraph; cache: IdentCache) =
+                graph: ModuleGraph) =
  let
    file = args[0].getStr
    line = args[1].getNum
@@ -210,7 +210,7 @@ proc executeEpc(cmd: IdeCmd, args: SexpNode;
  var dirtyfile = ""
  if len(args) > 3:
    dirtyfile = args[3].getStr(nil)
-  execute(cmd, file, dirtyfile, int(line), int(column), graph, cache)
+  execute(cmd, file, dirtyfile, int(line), int(column), graph)

 proc returnEpc(socket: Socket, uid: BiggestInt, s: SexpNode|string,
               return_symbol = "return") =
@@ -379,7 +379,7 @@ proc replEpc(x: ThreadParams) {.thread.} =
                         "unexpected call: " & epcAPI
      quit errMessage

-proc execCmd(cmd: string; graph: ModuleGraph; cache: IdentCache; cachedMsgs: CachedMsgs) =
+proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) =
  let conf = graph.config

  template sentinel() =
@@ -435,19 +435,19 @@ proc execCmd(cmd: string; graph: ModuleGraph; cache: IdentCache; cachedMsgs: Cac
  else:
    if conf.ideCmd == ideChk:
      for cm in cachedMsgs: errorHook(conf, cm.info, cm.msg, cm.sev)
-    execute(conf.ideCmd, orig, dirtyfile, line, col, graph, cache)
+    execute(conf.ideCmd, orig, dirtyfile, line, col, graph)
  sentinel()

-proc recompileFullProject(graph: ModuleGraph; cache: IdentCache) =
+proc recompileFullProject(graph: ModuleGraph) =
  #echo "recompiling full project"
  resetSystemArtifacts(graph)
  graph.vm = nil
  graph.resetAllModules()
  GC_fullcollect()
-  compileProject(graph, cache)
+  compileProject(graph)
  #echo GC_getStatistics()

-proc mainThread(graph: ModuleGraph; cache: IdentCache) =
+proc mainThread(graph: ModuleGraph) =
  let conf = graph.config
  if gLogging:
    for it in conf.searchPaths:
@@ -469,7 +469,7 @@ proc mainThread(graph: ModuleGraph; cache: IdentCache) =
    if hasData:
      conf.writelnHook = wrHook
      conf.suggestionResultHook = sugResultHook
-      execCmd(req, graph, cache, cachedMsgs)
+      execCmd(req, graph, cachedMsgs)
      idle = 0
    else:
      os.sleep 250
@@ -482,12 +482,12 @@ proc mainThread(graph: ModuleGraph; cache: IdentCache) =
      conf.structuredErrorHook = proc (conf: ConfigRef; info: TLineInfo; msg: string; sev: Severity) =
        cachedMsgs.add(CachedMsg(info: info, msg: msg, sev: sev))
      conf.suggestionResultHook = proc (s: Suggest) = discard
-      recompileFullProject(graph, cache)
+      recompileFullProject(graph)

 var
  inputThread: Thread[ThreadParams]

-proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
+proc mainCommand(graph: ModuleGraph) =
  let conf = graph.config
  clearPasses(graph)
  registerPass graph, verbosePass
@@ -509,7 +509,7 @@ proc mainCommand(graph: ModuleGraph; cache: IdentCache) =

  # compile the project before showing any input so that we already
  # can answer questions right away:
-  compileProject(graph, cache)
+  compileProject(graph)

  open(requests)
  open(results)
@@ -522,7 +522,7 @@ proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
                            (gPort, "sug \"" & conf.projectFull & "\":" & gAddress))
  of mcmdcon: createThread(inputThread, replCmdline,
                            (gPort, "con \"" & conf.projectFull & "\":" & gAddress))
-  mainThread(graph, cache)
+  mainThread(graph)
  joinThread(inputThread)
  close(requests)
  close(results)
@@ -632,6 +632,6 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =

    let graph = newModuleGraph(cache, conf)
    graph.suggestMode = true
-    mainCommand(graph, cache)
+    mainCommand(graph)

 handleCmdline(newIdentCache(), newConfigRef())