compiler bootstraps with new VM

2026-04-19 05:50:30 +00:00 · 2013-10-15 08:06:54 +02:00
parent 5659a1662e
commit 6a9baf3fd7
11 changed files with 141 additions and 64 deletions
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -14,7 +14,7 @@ import
  os, condsyms, rodread, rodwrite, times,
  wordrecg, sem, semdata, idents, passes, docgen, extccomp,
  cgen, jsgen, json, nversion,
-  platform, nimconf, importer, passaux, depends, evals, types, idgen,
+  platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
  tables, docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists,
  pretty

--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -14,7 +14,7 @@ import
  wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
  magicsys, parser, nversion, nimsets, semfold, importer,
  procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
-  semthreads, intsets, transf, evals, idgen, aliases, cgmeth, lambdalifting,
+  semthreads, intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
  evaltempl, patterns, parampatterns, sempass2

 # implementation
@@ -149,25 +149,26 @@ proc symNodeFromType(c: PContext, t: PType, info: TLineInfo): PNode =
  result = newSymNode(symFromType(t, info), info)
  result.typ = makeTypeDesc(c, t)

-proc createEvalContext(c: PContext, mode: TEvalMode): PEvalContext =
-  result = newEvalContext(c.module, mode)
-  result.getType = proc (n: PNode): PNode =
-    var e = tryExpr(c, n)
-    if e == nil:
-      result = symNodeFromType(c, errorType(c), n.info)
-    elif e.typ == nil:
-      result = newSymNode(getSysSym"void")
-    else:
-      result = symNodeFromType(c, e.typ, n.info)
+when false:
+  proc createEvalContext(c: PContext, mode: TEvalMode): PEvalContext =
+    result = newEvalContext(c.module, mode)
+    result.getType = proc (n: PNode): PNode =
+      var e = tryExpr(c, n)
+      if e == nil:
+        result = symNodeFromType(c, errorType(c), n.info)
+      elif e.typ == nil:
+        result = newSymNode(getSysSym"void")
+      else:
+        result = symNodeFromType(c, e.typ, n.info)

-  result.handleIsOperator = proc (n: PNode): PNode =
-    result = IsOpImpl(c, n)
+    result.handleIsOperator = proc (n: PNode): PNode =
+      result = IsOpImpl(c, n)

-proc evalConstExpr(c: PContext, module: PSym, e: PNode): PNode = 
-  result = evalConstExprAux(c.createEvalContext(emConst), module, nil, e)
+  proc evalConstExpr(c: PContext, module: PSym, e: PNode): PNode = 
+    result = evalConstExprAux(c.createEvalContext(emConst), module, nil, e)

-proc evalStaticExpr(c: PContext, module: PSym, e: PNode, prc: PSym): PNode = 
-  result = evalConstExprAux(c.createEvalContext(emStatic), module, prc, e)
+  proc evalStaticExpr(c: PContext, module: PSym, e: PNode, prc: PSym): PNode = 
+    result = evalConstExprAux(c.createEvalContext(emStatic), module, prc, e)

 proc semConstExpr(c: PContext, n: PNode): PNode =
  var e = semExprWithType(c, n)
@@ -176,7 +177,7 @@ proc semConstExpr(c: PContext, n: PNode): PNode =
    return n
  result = getConstExpr(c.module, e)
  if result == nil:
-    result = evalConstExpr(c, c.module, e)
+    result = evalConstExpr(c.module, e)
    if result == nil or result.kind == nkEmpty:
      if e.info != n.info:
        pushInfoContext(n.info)
@@ -222,10 +223,10 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
  if sym == c.p.owner:
    GlobalError(n.info, errRecursiveDependencyX, sym.name.s)

-  if c.evalContext == nil:
-    c.evalContext = c.createEvalContext(emStatic)
+  #if c.evalContext == nil:
+  #  c.evalContext = c.createEvalContext(emStatic)

-  result = evalMacroCall(c.evalContext, n, nOrig, sym)
+  result = evalMacroCall(c.module, n, nOrig, sym)
  if semCheck: result = semAfterMacroCall(c, result, sym)

 proc forceBool(c: PContext, n: PNode): PNode = 
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -13,7 +13,7 @@ import
  strutils, lists, intsets, options, lexer, ast, astalgo, trees, treetab,
  wordrecg, 
  ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, 
-  magicsys, nversion, nimsets, parser, times, passes, rodread, evals
+  magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef

 type 
  TOptionEntry* = object of lists.TListEntry # entries to put on a
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -631,18 +631,18 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
      call.add(a)
    #echo "NOW evaluating at compile time: ", call.renderTree
    if sfCompileTime in callee.flags:
-      result = evalStaticExpr(c, c.module, call, c.p.owner)
+      result = evalStaticExpr(c.module, call, c.p.owner)
      if result.isNil: 
        LocalError(n.info, errCannotInterpretNodeX, renderTree(call))
    else:
-      result = evalConstExpr(c, c.module, call)
+      result = evalConstExpr(c.module, call)
      if result.isNil: result = n
    #if result != n:
    #  echo "SUCCESS evaluated at compile time: ", call.renderTree

 proc semStaticExpr(c: PContext, n: PNode): PNode =
  let a = semExpr(c, n.sons[0])
-  result = evalStaticExpr(c, c.module, a, c.p.owner)
+  result = evalStaticExpr(c.module, a, c.p.owner)
  if result.isNil:
    LocalError(n.info, errCannotInterpretNodeX, renderTree(n))
    result = emptyNode
--- a/compiler/semmagic.nim
+++ b/compiler/semmagic.nim
@@ -32,6 +32,23 @@ proc semInstantiationInfo(c: PContext, n: PNode): PNode =
  result.add(filename)
  result.add(line)

+ 
+proc evalTypeTrait(trait, operand: PNode, context: PSym): PNode =
+  InternalAssert operand.kind == nkSym
+
+  let typ = operand.sym.typ.skipTypes({tyTypeDesc})
+  case trait.sym.name.s.normalize
+  of "name":
+    result = newStrNode(nkStrLit, typ.typeToString(preferName))
+    result.typ = newType(tyString, context)
+    result.info = trait.info
+  of "arity":    
+    result = newIntNode(nkIntLit, typ.n.len-1)
+    result.typ = newType(tyInt, context)
+    result.info = trait.info
+  else:
+    internalAssert false
+
 proc semTypeTraits(c: PContext, n: PNode): PNode =
  checkMinSonsLen(n, 2)
  internalAssert n.sons[1].kind == nkSym
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -1152,14 +1152,14 @@ proc setLine(n: PNode, info: TLineInfo) =
 proc semPragmaBlock(c: PContext, n: PNode): PNode =
  let pragmaList = n.sons[0]
  pragma(c, nil, pragmaList, exprPragmas)
-  result = semStmt(c, n.sons[1])
+  result = semExpr(c, n.sons[1])
  for i in 0 .. <pragmaList.len:
    if whichPragma(pragmaList.sons[i]) == wLine:
      setLine(result, pragmaList.sons[i].info)

 proc semStaticStmt(c: PContext, n: PNode): PNode =
  let a = semStmt(c, n.sons[0])
-  result = evalStaticExpr(c, c.module, a, c.p.owner)
+  result = evalStaticExpr(c.module, a, c.p.owner)
  if result.isNil:
    LocalError(n.info, errCannotInterpretNodeX, renderTree(n))
    result = emptyNode
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -12,7 +12,7 @@

 import
  strutils, ast, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, unsigned,
-  parser, vmdeps, idents
+  parser, vmdeps, idents, trees, renderer

 from semfold import leValueConv, ordinalValToString

@@ -231,13 +231,19 @@ proc compile(c: PCtx, s: PSym): int =
  result = vmgen.genProc(c, s)
  #c.echoCode

-proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
+proc regsContents(regs: TNodeSeq) =
+  for i in 0.. <regs.len:
+    echo "Register ", i
+    #debug regs[i]
+
+proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
  var pc = start
  var tos = tos
  var regs: TNodeSeq # alias to tos.slots for performance
  move(regs, tos.slots)
+  #echo "NEW RUN ------------------------"
  while true:
-    {.computedGoto.}
+    #{.computedGoto.}
    let instr = c.code[pc]
    let ra = instr.regA
    #echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra
@@ -248,12 +254,15 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
      pc = tos.comesFrom
      tos = tos.next
      let retVal = regs[0]
-      if tos.isNil: return retVal
+      if tos.isNil: 
+        #echo "RET ", retVal.rendertree
+        return retVal
      
      move(regs, tos.slots)
      assert c.code[pc].opcode in {opcIndCall, opcIndCallAsgn}
      if c.code[pc].opcode == opcIndCallAsgn:
        regs[c.code[pc].regA] = retVal
+        #echo "RET2 ", retVal.rendertree, " ", c.code[pc].regA
    of opcYldYoid: assert false
    of opcYldVal: assert false
    of opcAsgnInt:
@@ -508,7 +517,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
    of opcEcho:
      let rb = instr.regB
      for i in ra..ra+rb-1:
-        if regs[i].kind != nkStrLit: debug regs[i]
+        #if regs[i].kind != nkStrLit: debug regs[i]
        write(stdout, regs[i].strVal)
      writeln(stdout, "")
    of opcContainsSet:
@@ -535,6 +544,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
      let rc = instr.regC
      let prc = regs[rb].sym
      let newPc = compile(c, prc)
+      #echo "new pc ", newPc, " calling: ", prc.name.s
      var newFrame = PStackFrame(prc: prc, comesFrom: pc, next: tos)
      newSeq(newFrame.slots, prc.position)
      if not isEmptyType(prc.typ.sons[0]):
@@ -841,10 +851,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
      regs[ra].strVal = typ.typeToString(preferExported)
    inc pc

-proc execute(c: PCtx, start: int) =
+proc execute(c: PCtx, start: int): PNode =
  var tos = PStackFrame(prc: nil, comesFrom: 0, next: nil)
  newSeq(tos.slots, c.prc.maxSlots)
-  rawExecute(c, start, tos)
+  for i in 0 .. <c.prc.maxSlots: tos.slots[i] = newNode(nkEmpty)
+  result = rawExecute(c, start, tos)

 proc evalStmt*(c: PCtx, n: PNode) =
  let start = genStmt(c, n)
@@ -858,11 +869,24 @@ proc evalExpr*(c: PCtx, n: PNode): PNode =
  assert c.code[start].opcode != opcEof
  result = execute(c, start)

+# for now we share the 'globals' environment. XXX Coming soon: An API for
+# storing&loading the 'globals' environment to get what a component system
+# requires.
+var
+  globalCtx: PCtx
+
+proc setupGlobalCtx(module: PSym) =
+  if globalCtx.isNil: globalCtx = newCtx(module)
+  else: refresh(globalCtx, module)
+
 proc myOpen(module: PSym): PPassContext =
  #var c = newEvalContext(module, emRepl)
  #c.features = {allowCast, allowFFI, allowInfiniteLoops}
  #pushStackFrame(c, newStackFrame())
-  result = newCtx(module)
+
+  # XXX produce a new 'globals' environment here:
+  setupGlobalCtx(module)
+  result = globalCtx

 var oldErrorCount: int

@@ -875,17 +899,17 @@ proc myProcess(c: PPassContext, n: PNode): PNode =
    result = n
  oldErrorCount = msgs.gErrorCounter

-const vmPass* = makePass(myOpen, nil, myProcess, myProcess)
+const evalPass* = makePass(myOpen, nil, myProcess, myProcess)

-proc evalConstExprAux(module, prc: PSym, e: PNode, mode: TEvalMode): PNode = 
-  var p = newCtx(module)
-  var s = newStackFrame()
-  s.call = e
-  s.prc = prc
-  pushStackFrame(p, s)
-  result = tryEval(p, e)
-  if result != nil and result.kind == nkExceptBranch: result = nil
-  popStackFrame(p)
+proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode =
+  setupGlobalCtx(module)
+  var c = globalCtx
+  let start = genExpr(c, n)
+  assert c.code[start].opcode != opcEof
+  var tos = PStackFrame(prc: prc, comesFrom: 0, next: nil)
+  newSeq(tos.slots, c.prc.maxSlots)
+  for i in 0 .. <c.prc.maxSlots: tos.slots[i] = newNode(nkEmpty)
+  result = rawExecute(c, start, tos)

 proc evalConstExpr*(module: PSym, e: PNode): PNode = 
  result = evalConstExprAux(module, nil, e, emConst)
@@ -897,15 +921,18 @@ proc setupMacroParam(x: PNode): PNode =
  result = x
  if result.kind in {nkHiddenSubConv, nkHiddenStdConv}: result = result.sons[1]

-proc evalMacroCall(c: PEvalContext, n, nOrig: PNode, sym: PSym): PNode =
+var evalMacroCounter: int
+
+proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode =
  # XXX GlobalError() is ugly here, but I don't know a better solution for now
-  inc(evalTemplateCounter)
-  if evalTemplateCounter > 100:
+  inc(evalMacroCounter)
+  if evalMacroCounter > 100:
    GlobalError(n.info, errTemplateInstantiationTooNested)
+  setupGlobalCtx(module)
+  var c = globalCtx

  c.callsite = nOrig
-  let body = optBody(c, sym)
-  let start = genStmt(c, body)
+  let start = genProc(c, sym)

  var tos = PStackFrame(prc: sym, comesFrom: 0, next: nil)
  newSeq(tos.slots, c.prc.maxSlots)
@@ -917,8 +944,9 @@ proc evalMacroCall(c: PEvalContext, n, nOrig: PNode, sym: PSym): PNode =
  tos.slots[0] = newNodeIT(nkNilLit, n.info, sym.typ.sons[0])
  # setup parameters:
  for i in 1 .. < L: tos.slots[i] = setupMacroParam(n.sons[i])
-  rawExecute(c, start, tos)
-  result = tos.slots[0]
+  # temporary storage:
+  for i in L .. <c.prc.maxSlots: tos.slots[i] = newNode(nkEmpty)
+  result = rawExecute(c, start, tos)
  if cyclicTree(result): GlobalError(n.info, errCyclicTree)
-  dec(evalTemplateCounter)
+  dec(evalMacroCounter)
  c.callsite = nil
--- a/compiler/vmdef.nim
+++ b/compiler/vmdef.nim
@@ -165,12 +165,30 @@ type

  PEvalContext* = PCtx

+  TEvalMode* = enum           ## reason for evaluation
+    emRepl,                   ## evaluate because in REPL mode
+    emConst,                  ## evaluate for 'const' according to spec
+    emOptimize,               ## evaluate for optimization purposes (same as
+                              ## emConst?)
+    emStatic                  ## evaluate for enforced compile time eval
+                              ## ('static' context)
+
+  TSandboxFlag* = enum        ## what the evaluation engine should allow
+    allowCast,                ## allow unsafe language feature: 'cast'
+    allowFFI,                 ## allow the FFI
+    allowInfiniteLoops        ## allow endless loops
+  TSandboxFlags* = set[TSandboxFlag]
+
  
 proc newCtx*(module: PSym): PCtx =
  PCtx(code: @[], debug: @[],
    globals: newNode(nkStmtList), constants: newNode(nkStmtList), types: @[],
    prc: PProc(blocks: @[]), module: module)

+proc refresh*(c: PCtx, module: PSym) =
+  c.module = module
+  c.prc = PProc(blocks: @[])
+
 const
  firstABxInstr* = opcTJmp
  largeInstrs* = { # instructions which use 2 int32s instead of 1:
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -45,7 +45,7 @@ proc codeListing(c: PCtx, result: var string) =
    result.add("\n")
    inc i

-proc echoCode*(c: PCtx) =
+proc echoCode*(c: PCtx) {.deprecated.} =
  var buf = ""
  codeListing(c, buf)
  echo buf
@@ -494,7 +494,9 @@ proc genAddSubInt(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
    genBinaryABC(c, n, dest, opc)

 proc unused(n: PNode; x: TDest) {.inline.} =
-  if x >= 0: InternalError(n.info, "not unused")
+  if x >= 0: 
+    #debug(n)
+    InternalError(n.info, "not unused")

 proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) =  
  let tmp = c.genx(arg)
@@ -508,7 +510,7 @@ proc genCard(c: PCtx; n: PNode; dest: var TDest) =
  let tmp = c.genx(n.sons[1])
  if dest < 0: dest = c.getTemp(n.typ)
  c.genSetType(n.sons[1], tmp)
-  c.gABC(n, opc, dest, tmp)
+  c.gABC(n, opcCard, dest, tmp)
  c.freeTemp(tmp)

 proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
@@ -701,7 +703,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
  of mTypeTrait: 
    let tmp = c.genx(n.sons[1])
    if dest < 0: dest = c.getTemp(n.typ)
-    c.gABx(n, opcSetType, tmp, c.genType(n.sons[1]))
+    c.gABx(n, opcSetType, tmp, c.genType(n.sons[1].typ))
    c.gABC(n, opcTypeTrait, dest, tmp)
    c.freeTemp(tmp)
  of mIs:
@@ -1259,7 +1261,14 @@ proc optimizeJumps(c: PCtx; start: int) =
 proc genProc(c: PCtx; s: PSym): int =
  let x = s.ast.sons[optimizedCodePos]
  if x.kind == nkEmpty:
-    c.removeLastEof
+    #echo "GENERATING CODE FOR ", s.name.s
+    let last = c.code.len-1
+    var eofInstr: TInstr
+    if last >= 0 and c.code[last].opcode == opcEof:
+      eofInstr = c.code[last]
+      c.code.setLen(last)
+      c.debug.setLen(last)
+    #c.removeLastEof
    result = c.code.len+1 # skip the jump instruction
    s.ast.sons[optimizedCodePos] = newIntNode(nkIntLit, result)
    # thanks to the jmp we can add top level statements easily and also nest
@@ -1275,9 +1284,9 @@ proc genProc(c: PCtx; s: PSym): int =
    # generate final 'return' statement:
    c.gABC(body, opcRet)
    c.patch(procStart)
-    c.gABC(body, opcEof)
+    
+    c.gABC(body, opcEof, eofInstr.regA)
    s.position = c.prc.maxSlots
    c.prc = oldPrc
-    #c.echoCode
  else:
    result = x.intVal.int
--- a/todo.txt
+++ b/todo.txt
@@ -2,8 +2,9 @@ version 0.9.4
 =============

 - new VM:
-  - implement the glue to replace evals.nim
-  - implement missing magics
+  - new VM requires lambda lifting
+  - codegen for computed goto still wrong
+  - test and activate the jump optimizer
  - implement overflow checking
  - implement the FFI

--- a/web/news.txt
+++ b/web/news.txt
@@ -41,6 +41,9 @@ Compiler Additions
  over the generated code.
 - The compiler now supports a ``computedGoto`` pragma to support very fast
  dispatching for interpreters and the like.
+- The old evaluation engine has been replaced by a proper register based
+  virtual machine. This fixes numerous bugs for ``nimrod i`` and for macro
+  evaluation.


 Language Additions
@@ -55,7 +58,7 @@ Language Additions
  OOP-like syntactic sugar.
 - Added ``delegator pragma`` for handling calls to missing procs and fields at
  compile-time.
- Support for user-defined type classes have been added.
+- Support for user-defined type classes has been added.


 Tools improvements