completed VM support for incremental compilations

2026-06-09 13:18:11 +00:00 · 2018-06-04 01:01:08 +02:00
parent b5194f592c
commit 268260953c
6 changed files with 149 additions and 72 deletions
--- a/compiler/macrocacheimpl.nim
+++ b/compiler/macrocacheimpl.nim
@@ -11,39 +11,69 @@

 import lineinfos, ast, modulegraphs, vmdef, magicsys

-proc genCall3(g: ModuleGraph; m: TMagic; s: string; a, b, c: PNode): PNode =
-  newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b, c))
-
-proc genCall2(g: ModuleGraph; m: TMagic; s: string; a, b: PNode): PNode =
-  newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b))
-
-template nodeFrom(s: string): PNode =
-  var res = newStrNode(s, info)
-  res.typ = getSysType(g, info, tyString)
-  res
-
-template nodeFrom(i: BiggestInt): PNode =
-  var res = newIntNode(i, info)
-  res.typ = getSysType(g, info, tyInt)
-  res
-
-template nodeFrom(n: PNode): PNode = copyTree(n)
-
-template record(call) =
-  g.recordStmt(g, c.module, call)
-
 proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
-  let g = c.graph
-  record genCall2(mNccInc, "inc", nodeFrom key, nodeFrom by)
+  var recorded = newNodeI(nkCommentStmt, info)
+  recorded.add newStrNode("inc", info)
+  recorded.add newStrNode(key, info)
+  recorded.add newIntNode(nkIntLit, by)
+  c.graph.recordStmt(c.graph, c.module, recorded)

 proc recordPut*(c: PCtx; info: TLineInfo; key: string; k: string; val: PNode) =
-  let g = c.graph
-  record genCall3(mNctPut, "[]=", nodeFrom key, nodeFrom k, nodeFrom val)
+  var recorded = newNodeI(nkCommentStmt, info)
+  recorded.add newStrNode("put", info)
+  recorded.add newStrNode(key, info)
+  recorded.add newStrNode(k, info)
+  recorded.add copyTree(val)
+  c.graph.recordStmt(c.graph, c.module, recorded)

 proc recordAdd*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
-  let g = c.graph
-  record genCall2(mNcsAdd, "add", nodeFrom key, nodeFrom val)
+  var recorded = newNodeI(nkCommentStmt, info)
+  recorded.add newStrNode("add", info)
+  recorded.add newStrNode(key, info)
+  recorded.add copyTree(val)
+  c.graph.recordStmt(c.graph, c.module, recorded)

 proc recordIncl*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
-  let g = c.graph
-  record genCall2(mNcsIncl, "incl", nodeFrom key, nodeFrom val)
+  var recorded = newNodeI(nkCommentStmt, info)
+  recorded.add newStrNode("incl", info)
+  recorded.add newStrNode(key, info)
+  recorded.add copyTree(val)
+  c.graph.recordStmt(c.graph, c.module, recorded)
+
+when false:
+  proc genCall3(g: ModuleGraph; m: TMagic; s: string; a, b, c: PNode): PNode =
+    newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b, c))
+
+  proc genCall2(g: ModuleGraph; m: TMagic; s: string; a, b: PNode): PNode =
+    newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b))
+
+  template nodeFrom(s: string): PNode =
+    var res = newStrNode(s, info)
+    res.typ = getSysType(g, info, tyString)
+    res
+
+  template nodeFrom(i: BiggestInt): PNode =
+    var res = newIntNode(i, info)
+    res.typ = getSysType(g, info, tyInt)
+    res
+
+  template nodeFrom(n: PNode): PNode = copyTree(n)
+
+  template record(call) =
+    g.recordStmt(g, c.module, call)
+
+  proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
+    let g = c.graph
+    record genCall2(mNccInc, "inc", nodeFrom key, nodeFrom by)
+
+  proc recordPut*(c: PCtx; info: TLineInfo; key: string; k: string; val: PNode) =
+    let g = c.graph
+    record genCall3(mNctPut, "[]=", nodeFrom key, nodeFrom k, nodeFrom val)
+
+  proc recordAdd*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
+    let g = c.graph
+    record genCall2(mNcsAdd, "add", nodeFrom key, nodeFrom val)
+
+  proc recordIncl*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
+    let g = c.graph
+    record genCall2(mNcsIncl, "incl", nodeFrom key, nodeFrom val)
--- a/compiler/modulegraphs.nim
+++ b/compiler/modulegraphs.nim
@@ -26,7 +26,7 @@
 ##

 import ast, intsets, tables, options, lineinfos, hashes, idents,
-  incremental
+  incremental, btrees

 type
  ModuleGraph* = ref object
@@ -59,6 +59,9 @@ type
    importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
    includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
    recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}
+    cacheSeqs*: Table[string, PNode] # state that is shared to suppor the 'macrocache' API
+    cacheCounters*: Table[string, BiggestInt]
+    cacheTables*: Table[string, BTree[string, PNode]]

 proc hash*(x: FileIndex): Hash {.borrow.}

@@ -90,6 +93,9 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
  init(result.incr)
  result.recordStmt = proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.} =
    discard
+  result.cacheSeqs = initTable[string, PNode]()
+  result.cacheCounters = initTable[string, BiggestInt]()
+  result.cacheTables = initTable[string, BTree[string, PNode]]()

 proc resetAllModules*(g: ModuleGraph) =
  initStrTable(packageSyms)
--- a/compiler/rodimpl.nim
+++ b/compiler/rodimpl.nim
@@ -10,7 +10,8 @@
 ## This module implements the new compilation cache.

 import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
-  renderer, rodutils, idents, astalgo, btrees, magicsys, cgmeth, extccomp, vm
+  renderer, rodutils, idents, astalgo, btrees, magicsys, cgmeth, extccomp,
+  btrees, trees

 ## Todo:
 ## - Make conditional symbols and the configuration part of a module's
@@ -757,9 +758,10 @@ proc loadModuleSymTab(g; module: PSym) =
    g.systemModule = module

 proc replay(g: ModuleGraph; module: PSym; n: PNode) =
+  # XXX check if we need to replay nkStaticStmt here.
  case n.kind
-  of nkStaticStmt:
-    evalStaticStmt(module, g, n[0], module)
+  #of nkStaticStmt:
+    #evalStaticStmt(module, g, n[0], module)
    #of nkVarSection, nkLetSection:
    #  nkVarSections are already covered by the vmgen which produces nkStaticStmt
  of nkMethodDef:
@@ -786,6 +788,39 @@ proc replay(g: ModuleGraph; module: PSym; n: PNode) =
        extccomp.addExternalFileToCompile(g.config, cf)
      of "link":
        extccomp.addExternalFileToLink(g.config, n[1].strVal)
+      of "inc":
+        let destKey = n[1].strVal
+        let by = n[2].intVal
+        let v = getOrDefault(g.cacheCounters, destKey)
+        g.cacheCounters[destKey] = v+by
+      of "put":
+        let destKey = n[1].strVal
+        let key = n[2].strVal
+        let val = n[3]
+        if not contains(g.cacheTables, destKey):
+          g.cacheTables[destKey] = initBTree[string, PNode]()
+        if not contains(g.cacheTables[destKey], key):
+          g.cacheTables[destKey].add(key, val)
+        else:
+          internalError(g.config, n.info, "key already exists: " & key)
+      of "incl":
+        let destKey = n[1].strVal
+        let val = n[2]
+        if not contains(g.cacheSeqs, destKey):
+          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
+        else:
+          block search:
+            for existing in g.cacheSeqs[destKey]:
+              if exprStructuralEquivalent(existing, val, strictSymEquality=true):
+                break search
+            g.cacheSeqs[destKey].add val
+      of "add":
+        let destKey = n[1].strVal
+        let val = n[2]
+        if not contains(g.cacheSeqs, destKey):
+          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
+        else:
+          g.cacheSeqs[destKey].add val
      else:
        internalAssert g.config, false
  of nkImportStmt:
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -1576,86 +1576,98 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
    of opcNccValue:
      decodeB(rkInt)
      let destKey = regs[rb].node.strVal
-      regs[ra].intVal = getOrDefault(c.cacheCounters, destKey)
+      regs[ra].intVal = getOrDefault(c.graph.cacheCounters, destKey)
    of opcNccInc:
-      let rb = instr.regB
+      let g = c.graph
      let destKey = regs[ra].node.strVal
-      let by = regs[rb].intVal
-      let v = getOrDefault(c.cacheCounters, destKey)
-      c.cacheCounters[destKey] = v+by
+      let by = regs[instr.regB].intVal
+      let v = getOrDefault(g.cacheCounters, destKey)
+      g.cacheCounters[destKey] = v+by
      recordInc(c, c.debug[pc], destKey, by)
    of opcNcsAdd:
+      let g = c.graph
      let destKey = regs[ra].node.strVal
      let val = regs[instr.regB].node
-      if not contains(c.cacheSeqs, destKey):
-        c.cacheSeqs[destKey] = newTree(nkStmtList, val)
+      if not contains(g.cacheSeqs, destKey):
+        g.cacheSeqs[destKey] = newTree(nkStmtList, val)
        # newNodeI(nkStmtList, c.debug[pc])
      else:
-        c.cacheSeqs[destKey].add val
+        g.cacheSeqs[destKey].add val
      recordAdd(c, c.debug[pc], destKey, val)
    of opcNcsIncl:
+      let g = c.graph
      let destKey = regs[ra].node.strVal
      let val = regs[instr.regB].node
-      if not contains(c.cacheSeqs, destKey):
-        c.cacheSeqs[destKey] = newTree(nkStmtList, val)
+      if not contains(g.cacheSeqs, destKey):
+        g.cacheSeqs[destKey] = newTree(nkStmtList, val)
      else:
        block search:
-          for existing in c.cacheSeqs[destKey]:
+          for existing in g.cacheSeqs[destKey]:
            if exprStructuralEquivalent(existing, val, strictSymEquality=true):
              break search
-          c.cacheSeqs[destKey].add val
+          g.cacheSeqs[destKey].add val
      recordIncl(c, c.debug[pc], destKey, val)
    of opcNcsLen:
+      let g = c.graph
      decodeB(rkInt)
      let destKey = regs[rb].node.strVal
      regs[ra].intVal =
-        if contains(c.cacheSeqs, destKey): c.cacheSeqs[destKey].len else: 0
+        if contains(g.cacheSeqs, destKey): g.cacheSeqs[destKey].len else: 0
    of opcNcsAt:
+      let g = c.graph
      decodeBC(rkNode)
      let idx = regs[rc].intVal
      let destKey = regs[rb].node.strVal
-      if contains(c.cacheSeqs, destKey) and idx <% c.cacheSeqs[destKey].len:
-        regs[ra].node = c.cacheSeqs[destKey][idx.int]
+      if contains(g.cacheSeqs, destKey) and idx <% g.cacheSeqs[destKey].len:
+        regs[ra].node = g.cacheSeqs[destKey][idx.int]
      else:
        stackTrace(c, tos, pc, errIndexOutOfBounds)
    of opcNctPut:
+      let g = c.graph
      let destKey = regs[ra].node.strVal
      let key = regs[instr.regB].node.strVal
      let val = regs[instr.regC].node
-      if not contains(c.cacheTables, destKey):
-        c.cacheTables[destKey] = initBTree[string, PNode]()
-      if not contains(c.cacheTables[destKey], key):
-        c.cacheTables[destKey].add(key, val)
+      if not contains(g.cacheTables, destKey):
+        g.cacheTables[destKey] = initBTree[string, PNode]()
+      if not contains(g.cacheTables[destKey], key):
+        g.cacheTables[destKey].add(key, val)
        recordPut(c, c.debug[pc], destKey, key, val)
      else:
        stackTrace(c, tos, pc, "key already exists: " & key)
    of opcNctLen:
+      let g = c.graph
      decodeB(rkInt)
      let destKey = regs[rb].node.strVal
      regs[ra].intVal =
-        if contains(c.cacheTables, destKey): c.cacheTables[destKey].len else: 0
+        if contains(g.cacheTables, destKey): g.cacheTables[destKey].len else: 0
    of opcNctGet:
+      let g = c.graph
      decodeBC(rkNode)
      let destKey = regs[rb].node.strVal
      let key = regs[rc].node.strVal
-      if contains(c.cacheTables, destKey):
-        if contains(c.cacheTables[destKey], key):
-          regs[ra].node = getOrDefault(c.cacheTables[destKey], key)
+      if contains(g.cacheTables, destKey):
+        if contains(g.cacheTables[destKey], key):
+          regs[ra].node = getOrDefault(g.cacheTables[destKey], key)
        else:
          stackTrace(c, tos, pc, "key does not exist: " & key)
      else:
        stackTrace(c, tos, pc, "key does not exist: " & destKey)
    of opcNctHasNext:
+      let g = c.graph
      decodeBC(rkInt)
      let destKey = regs[rb].node.strVal
      regs[ra].intVal =
-        btrees.hasNext(c.cacheTables[destKey], regs[rc].intVal.int) else: 0
+        if g.cacheTables.contains(destKey):
+          ord(btrees.hasNext(g.cacheTables[destKey], regs[rc].intVal.int))
+        else:
+          0
    of opcNctNext:
+      let g = c.graph
      decodeBC(rkNode)
      let destKey = regs[rb].node.strVal
      let index = regs[rc].intVal
-      if contains(c.cacheTables, destKey):
-        let (k, v, nextIndex) = btrees.next(c.cacheTables[destKey], index.int)
+      if contains(g.cacheTables, destKey):
+        let (k, v, nextIndex) = btrees.next(g.cacheTables[destKey], index.int)
        regs[ra].node = newTree(nkTupleConstr, newStrNode(k, c.debug[pc]), v,
                                newIntNode(nkIntLit, nextIndex))
      else:
--- a/compiler/vmdef.nim
+++ b/compiler/vmdef.nim
@@ -213,9 +213,6 @@ type
    config*: ConfigRef
    graph*: ModuleGraph
    oldErrorCount*: int
-    cacheSeqs*: Table[string, PNode]
-    cacheCounters*: Table[string, BiggestInt]
-    cacheTables*: Table[string, BTree[string, PNode]]

  TPosition* = distinct int

@@ -226,10 +223,7 @@ proc newCtx*(module: PSym; cache: IdentCache; g: ModuleGraph): PCtx =
    globals: newNode(nkStmtListExpr), constants: newNode(nkStmtList), types: @[],
    prc: PProc(blocks: @[]), module: module, loopIterations: MaxLoopIterations,
    comesFromHeuristic: unknownLineInfo(), callbacks: @[], errorFlag: "",
-    cache: cache, config: g.config, graph: g,
-    cacheSeqs: initTable[string, PNode]()
-    cacheCounters: initTable[string, BiggestInt]()
-    cacheTables: initTable[string, BTree[string, PNode]]())
+    cache: cache, config: g.config, graph: g)

 proc refresh*(c: PCtx, module: PSym) =
  c.module = module
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -804,12 +804,12 @@ proc genIntCast(c: PCtx; n: PNode; dest: var TDest) =
  else:
    globalError(c.config, n.info, "VM is only allowed to 'cast' between integers of same size")

-proc genVoidABC(c: PCtx, n: PNode, dest: TRegister, opcode: TOpcode)
+proc genVoidABC(c: PCtx, n: PNode, dest: TRegister, opcode: TOpcode) =
  unused(c, n, dest)
  var
-    tmp1 = c.genx(n.sons[1])
-    tmp2 = c.genx(n.sons[2])
-    tmp3 = c.genx(n.sons[3])
+    tmp1 = c.genx(n[1])
+    tmp2 = c.genx(n[2])
+    tmp3 = c.genx(n[3])
  c.gABC(n, opcode, tmp1, tmp2, tmp3)
  c.freeTemp(tmp1)
  c.freeTemp(tmp2)
@@ -1087,8 +1087,8 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =

  of mNccValue: genUnaryABC(c, n, dest, opcNccValue)
  of mNccInc: genBinaryABC(c, n, dest, opcNccInc)
-  of mNcsAdd: genBinaryABC(c, n, dest, opcNcsAdd
-  of mNcsIncl: genBinaryABC(c, n, dest, opcNcsIncl
+  of mNcsAdd: genBinaryABC(c, n, dest, opcNcsAdd)
+  of mNcsIncl: genBinaryABC(c, n, dest, opcNcsIncl)
  of mNcsLen: genUnaryABC(c, n, dest, opcNcsLen)
  of mNcsAt: genBinaryABC(c, n, dest, opcNcsAt)
  of mNctPut: genVoidABC(c, n, dest, opcNctPut)