VM now supports lambda lifting

compiler/ast.nim

@@ -357,6 +357,7 @@ type
     nfSem       # node has been checked for semantics
     nfDelegate  # the call can use a delegator
     nfExprCall  # this is an attempt to call a regular expression
+    nfIsRef     # this node is a 'ref' node; used for the VM
 
   TNodeFlags* = set[TNodeFlag]
   TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 23)
@@ -785,7 +786,7 @@ const
   ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType, skIterator, 
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
-                                      nfAllConst, nfDelegate}
+                                      nfAllConst, nfDelegate, nfIsRef}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2

compiler/lambdalifting.nim

@@ -207,7 +207,7 @@ proc newCall(a, b: PSym): PNode =
 
 proc addHiddenParam(routine: PSym, param: PSym) =
   var params = routine.ast.sons[paramsPos]
-  param.position = params.len
+  param.position = params.len-1
   addSon(params, newSymNode(param))
   incl(routine.typ.flags, tfCapturesEnv)
   #echo "produced environment: ", param.id, " for ", routine.name.s
@@ -549,6 +549,8 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode =
       if x != nil: n.sons[i] = x
 
 proc liftLambdas*(fn: PSym, body: PNode): PNode =
+  # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
+  # the transformation even when compiling to JS ...
   if body.kind == nkEmpty or gCmd == cmdCompileToJS:
     # ignore forward declaration:
     result = body

compiler/transf.nim

@@ -735,12 +735,9 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
   if nfTransf in n.flags or prc.kind in {skTemplate}:
     result = n
   else:
-    #when useEffectSystem: trackProc(prc, n)
     var c = openTransf(module, "")
     result = processTransf(c, n, prc)
-    if prc.kind != skMacro:
-      # XXX no closures yet for macros:
-      result = liftLambdas(prc, result)
+    result = liftLambdas(prc, result)
     if prc.kind == skIterator and prc.typ.callConv == ccClosure:
       result = lambdalifting.liftIterator(prc, result)
     incl(result.flags, nfTransf)

compiler/vm.nim

@@ -133,6 +133,27 @@ proc moveConst(x, y: PNode) =
 # of PNimrodNode:
 template asgnRef(x, y: expr) = moveConst(x, y)
 
+proc copyValue(src: PNode): PNode =
+  if src == nil or nfIsRef in src.flags:
+    return src
+  result = newNode(src.kind)
+  result.info = src.info
+  result.typ = src.typ
+  result.flags = src.flags * PersistentNodeFlags
+  when defined(useNodeIds):
+    if result.id == nodeIdToDebug:
+      echo "COMES FROM ", src.id
+  case src.Kind
+  of nkCharLit..nkUInt64Lit: result.intVal = src.intVal
+  of nkFloatLit..nkFloat128Lit: result.floatVal = src.floatVal
+  of nkSym: result.sym = src.sym
+  of nkIdent: result.ident = src.ident
+  of nkStrLit..nkTripleStrLit: result.strVal = src.strVal
+  else:
+    newSeq(result.sons, sonsLen(src))
+    for i in countup(0, sonsLen(src) - 1):
+      result.sons[i] = copyValue(src.sons[i])
+
 proc asgnComplex(x, y: PNode) =
   if x.kind != y.kind:
     myreset(x)
@@ -149,7 +170,7 @@ proc asgnComplex(x, y: PNode) =
     else: x.sons[0] = y.sons[0]
   else:
     if x.kind notin {nkEmpty..nkNilLit}:
-      let y = y.copyTree
+      let y = y.copyValue
       for i in countup(0, sonsLen(y) - 1): addSon(x, y.sons[i])
 
 template getstr(a: expr): expr =
@@ -306,9 +327,9 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
     of opcAsgnRef:
       asgnRef(regs[ra], regs[instr.regB])
     of opcWrGlobalRef:
-      asgnRef(c.globals.sons[instr.regBx-wordExcess-1], regs[ra].skipMeta)
+      asgnRef(c.globals.sons[instr.regBx-wordExcess-1], regs[ra])
     of opcWrGlobal:
-      asgnComplex(c.globals.sons[instr.regBx-wordExcess-1], regs[ra].skipMeta)
+      asgnComplex(c.globals.sons[instr.regBx-wordExcess-1], regs[ra])
     of opcLdArr:
       # a = b[c]
       let rb = instr.regB
@@ -326,12 +347,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       let rb = instr.regB
       let rc = instr.regC
       let idx = regs[rb].intVal
-      asgnComplex(regs[ra].sons[idx.int], regs[rc].skipMeta)
+      asgnComplex(regs[ra].sons[idx.int], regs[rc])
     of opcWrArrRef:
       let rb = instr.regB
       let rc = instr.regC
       let idx = regs[rb].intVal
-      asgnRef(regs[ra].sons[idx.int], regs[rc].skipMeta)
+      asgnRef(regs[ra].sons[idx.int], regs[rc])
     of opcLdObj:
       # a = b.c
       let rb = instr.regB
@@ -343,11 +364,16 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       # a.b = c
       let rb = instr.regB
       let rc = instr.regC
-      asgnComplex(regs[ra].sons[rb], regs[rc].skipMeta)
+      #if regs[ra].isNil or regs[ra].sons.isNil or rb >= len(regs[ra]):
+      #  debug regs[ra]
+      #  debug regs[rc]
+      #  echo "RB ", rb
+      #  internalError(c.debug[pc], "argl")
+      asgnComplex(regs[ra].sons[rb], regs[rc])
     of opcWrObjRef:
       let rb = instr.regB
       let rc = instr.regC
-      asgnRef(regs[ra].sons[rb], regs[rc].skipMeta)
+      asgnRef(regs[ra].sons[rb], regs[rc])
     of opcWrStrIdx:
       decodeBC(nkStrLit)
       let idx = regs[rb].intVal.int
@@ -580,7 +606,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       # dest = call regStart, n; where regStart = fn, arg1, ...
       let rb = instr.regB
       let rc = instr.regC
-      let prc = regs[rb].sym
+      let isClosure = regs[rb].kind == nkPar
+      let prc = if not isClosure: regs[rb].sym else: regs[rb].sons[0].sym
       let newPc = compile(c, prc)
       #echo "new pc ", newPc, " calling: ", prc.name.s
       var newFrame = PStackFrame(prc: prc, comesFrom: pc, next: tos)
@@ -590,8 +617,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       # pass every parameter by var (the language definition allows this):
       for i in 1 .. rc-1:
         newFrame.slots[i] = regs[rb+i]
+      if isClosure:
+        newFrame.slots[rc] = regs[rb].sons[1]
       # allocate the temporaries:
-      for i in rc .. <prc.position:
+      for i in rc+ord(isClosure) .. <prc.position:
         newFrame.slots[i] = newNode(nkEmpty)
       tos = newFrame
       move(regs, newFrame.slots)
@@ -656,6 +685,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
     of opcNew:
       let typ = c.types[instr.regBx - wordExcess]
       regs[ra] = getNullValue(typ, regs[ra].info)
+      regs[ra].flags.incl nfIsRef
     of opcNewSeq:
       let typ = c.types[instr.regBx - wordExcess]
       inc pc
@@ -724,6 +754,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       setMeta(regs[ra], regs[rb].skipMeta.sons[1])
     of opcNChild:
       decodeBC(nkMetaNode)
+      if regs[rb].kind != nkMetaNode:
+        internalError(c.debug[pc], "no MetaNode")
       setMeta(regs[ra], regs[rb].uast.sons[regs[rc].intVal.int])
     of opcNSetChild:
       decodeBC(nkMetaNode)
@@ -886,19 +918,17 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
          regs[rb].kind in {nkStrLit..nkTripleStrLit}:
         dest.strVal = regs[rb].strVal
       else:
-        #c.echoCode
-        #debug regs[ra]
-        #debug regs[rb]
         stackTrace(c, tos, pc, errFieldXNotFound, "strVal")
     of opcNNewNimNode:
       decodeBC(nkMetaNode)
       var k = regs[rb].intVal
-      if k < 0 or k > ord(high(TNodeKind)): 
+      if k < 0 or k > ord(high(TNodeKind)) or k == ord(nkMetaNode):
         internalError(c.debug[pc],
           "request to create a NimNode of invalid kind")
       let cc = regs[rc].skipMeta
       setMeta(regs[ra], newNodeI(TNodeKind(int(k)), 
         if cc.kind == nkNilLit: c.debug[pc] else: cc.info))
+      regs[ra].sons[0].flags.incl nfIsRef
     of opcNCopyNimNode:
       decodeB(nkMetaNode)
       setMeta(regs[ra], copyNode(regs[rb]))
@@ -1006,6 +1036,7 @@ proc setupMacroParam(x: PNode): PNode =
   result = x
   if result.kind in {nkHiddenSubConv, nkHiddenStdConv}: result = result.sons[1]
   let y = result
+  y.flags.incl nfIsRef
   result = newNode(nkMetaNode)
   result.add y
 
@@ -1039,6 +1070,5 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode =
   if cyclicTree(result): GlobalError(n.info, errCyclicTree)
   dec(evalMacroCounter)
   if result != nil:
-    internalAssert result.kind == nkMetaNode
-    result = result.sons[0]
+    result = result.skipMeta
   c.callsite = nil

compiler/vmgen.nim

@@ -948,8 +948,15 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
   of tyFloat..tyFloat128: 
     result = newNodeIt(nkFloatLit, info, t)
   of tyVar, tyPointer, tyPtr, tyCString, tySequence, tyString, tyExpr, 
-     tyStmt, tyTypeDesc, tyProc, tyRef:
+     tyStmt, tyTypeDesc, tyRef:
     result = newNodeIT(nkNilLit, info, t)
+  of tyProc:
+    if t.callConv != ccClosure:
+      result = newNodeIT(nkNilLit, info, t)
+    else:
+      result = newNodeIT(nkPar, info, t)
+      result.add(newNodeIT(nkNilLit, info, t))
+      result.add(newNodeIT(nkNilLit, info, t))
   of tyObject: 
     result = newNodeIT(nkPar, info, t)
     getNullValueAux(t.n, result)
@@ -1071,7 +1078,8 @@ proc genObjConstr(c: PCtx, n: PNode, dest: var TDest) =
 
 proc genTupleConstr(c: PCtx, n: PNode, dest: var TDest) =
   if dest < 0: dest = c.getTemp(n.typ)
-  var idx = getTemp(c, getSysType(tyInt))
+  c.gABx(n, opcLdNull, dest, c.genType(n.typ))
+  # XXX x = (x.old, 22)  produces wrong code ... stupid self assignments
   for i in 0.. <n.len:
     let it = n.sons[i]
     if it.kind == nkExprColonExpr:
@@ -1082,10 +1090,8 @@ proc genTupleConstr(c: PCtx, n: PNode, dest: var TDest) =
       c.freeTemp(idx)
     else:
       let tmp = c.genx(it)
-      c.gABx(it, opcLdImmInt, idx, i)
-      c.gABC(it, whichAsgnOpc(it, opcWrObj), dest, idx, tmp)
+      c.gABC(it, whichAsgnOpc(it, opcWrObj), dest, i.TRegister, tmp)
       c.freeTemp(tmp)
-  c.freeTemp(idx)
 
 proc genProc*(c: PCtx; s: PSym): int
 
@@ -1280,7 +1286,8 @@ proc optimizeJumps(c: PCtx; start: int) =
 proc genProc(c: PCtx; s: PSym): int =
   let x = s.ast.sons[optimizedCodePos]
   if x.kind == nkEmpty:
-    #echo "GENERATING CODE FOR ", s.name.s
+    #if s.name.s == "outterMacro" or s.name.s == "innerProc":
+    #  echo "GENERATING CODE FOR ", s.name.s
     let last = c.code.len-1
     var eofInstr: TInstr
     if last >= 0 and c.code[last].opcode == opcEof:
@@ -1299,6 +1306,11 @@ proc genProc(c: PCtx; s: PSym): int =
     c.prc = p
     # iterate over the parameters and allocate space for them:
     genParams(c, s.typ.n)
+    if tfCapturesEnv in s.typ.flags:
+      #let env = s.ast.sons[paramsPos].lastSon.sym
+      #assert env.position == 2
+      c.prc.slots[c.prc.maxSlots] = (inUse: true, kind: slotFixedLet)
+      inc c.prc.maxSlots
     gen(c, body)
     # generate final 'return' statement:
     c.gABC(body, opcRet)
@@ -1306,8 +1318,9 @@ proc genProc(c: PCtx; s: PSym): int =
     c.gABC(body, opcEof, eofInstr.regA)
     c.optimizeJumps(result)
     s.position = c.prc.maxSlots
-    #if s.name.s == "temp":
+    #if s.name.s == "innerProc":
     #  c.echoCode
+    #  echo renderTree(body)
     c.prc = oldPrc
   else:
     c.prc.maxSlots = s.position

todo.txt

@@ -3,7 +3,6 @@ version 0.9.4
 
 - new VM:
   - tcntseq fails
-  - new VM requires lambda lifting
   - implement overflow checking
   - implement the FFI