FFI at compiletime improvements

compiler/astalgo.nim

@@ -429,7 +429,7 @@ proc debugTree(n: PNode, indent: int, maxRecDepth: int): PRope =
                    [istr, makeYamlString($n.kind)])
     if maxRecDepth != 0: 
       case n.kind
-      of nkCharLit..nkInt64Lit: 
+      of nkCharLit..nkUInt64Lit:
         appf(result, ",$N$1\"intVal\": $2", [istr, toRope(n.intVal)])
       of nkFloatLit, nkFloat32Lit, nkFloat64Lit: 
         appf(result, ",$N$1\"floatVal\": $2", 

compiler/evalffi.nim

@@ -156,7 +156,10 @@ proc packObject(x: PNode, typ: PType, res: pointer) =
       pack(it, field.typ, res +! field.offset)
     else:
       GlobalError(x.info, "cannot pack unnamed tuple")
-      
+
+const maxPackDepth = 20
+var packRecCheck = 0
+
 proc pack(v: PNode, typ: PType, res: pointer) =
   template awr(T, v: expr) {.immediate, dirty.} =
     wr(T, res, v)
@@ -186,12 +189,14 @@ proc pack(v: PNode, typ: PType, res: pointer) =
   of tyFloat32: awr(float32, v.floatVal)
   of tyFloat64: awr(float64, v.floatVal)
   
-  of tyPointer, tyProc:
+  of tyPointer, tyProc,  tyCString, tyString:
     if v.kind == nkNilLit:
       # nothing to do since the memory is 0 initialized anyway
       nil
     elif v.kind == nkPtrLit:
       awr(pointer, cast[pointer](v.intVal))
+    elif v.kind in {nkStrLit..nkTripleStrLit}:
+      awr(cstring, cstring(v.strVal))
     else:
       GlobalError(v.info, "cannot map pointer/proc value to FFI")
   of tyPtr, tyRef, tyVar:
@@ -201,15 +206,13 @@ proc pack(v: PNode, typ: PType, res: pointer) =
     elif v.kind == nkPtrLit:
       awr(pointer, cast[pointer](v.intVal))
     else:
+      if packRecCheck > maxPackDepth:
+        packRecCheck = 0
+        GlobalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
+      inc packRecCheck
       pack(v.sons[0], typ.sons[0], res +! sizeof(pointer))
+      dec packRecCheck
       awr(pointer, res +! sizeof(pointer))
-  of tyCString, tyString:
-    if v.kind == nkNilLit:
-      nil
-    elif v.kind in {nkStrLit..nkTripleStrLit}:
-      awr(cstring, cstring(v.strVal))
-    else:
-      GlobalError(v.info, "cannot map string value to FFI")
   of tyArray, tyArrayConstr:
     let baseSize = typ.sons[1].getSize
     for i in 0 .. <v.len:
@@ -236,6 +239,8 @@ proc unpackObjectAdd(x: pointer, n, result: PNode) =
     var pair = newNodeI(nkExprColonExpr, result.info, 2)
     pair.sons[0] = n
     pair.sons[1] = unpack(x +! n.sym.offset, n.sym.typ, nil)
+    #echo "offset: ", n.sym.name.s, " ", n.sym.offset
+    result.add pair
   else: nil
 
 proc unpackObject(x: pointer, typ: PType, n: PNode): PNode =
@@ -295,8 +300,10 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
       # check we have the right field:
       result = n
       if result.kind.canonNodeKind != k.canonNodeKind:
-        echo "expected ", k, " but got ", result.kind
-        GlobalError(n.info, "cannot map value from FFI")
+        #echo "expected ", k, " but got ", result.kind
+        #debug result
+        return newNodeI(nkExceptBranch, n.info)
+        #GlobalError(n.info, "cannot map value from FFI")
     result.field = v
 
   template setNil() =
@@ -315,7 +322,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
   
   case typ.kind
   of tyBool: awi(nkIntLit, rd(bool, x).ord)
-  of tyChar: awi(nkIntLit, rd(char, x).ord)
+  of tyChar: awi(nkCharLit, rd(char, x).ord)
   of tyInt:  awi(nkIntLit, rd(int, x))
   of tyInt8: awi(nkInt8Lit, rd(int8, x))
   of tyInt16: awi(nkInt16Lit, rd(int16, x))
@@ -341,6 +348,10 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
     let p = rd(pointer, x)
     if p.isNil:
       setNil()
+    elif n != nil and n.kind == nkStrLit:
+      # we passed a string literal as a pointer; however strings are already
+      # in their unboxed representation so nothing it to be unpacked:
+      result = n
     else:
       awi(nkPtrLit, cast[TAddress](p))
   of tyPtr, tyRef, tyVar:
@@ -350,7 +361,9 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
     elif n == nil or n.kind == nkPtrLit:
       awi(nkPtrLit, cast[TAddress](p))
     elif n != nil and n.len == 1:
-      n.sons[0] = unpack(rd(pointer, x), typ.sons[0], n.sons[0])
+      internalAssert n.kind == nkRefTy
+      n.sons[0] = unpack(p, typ.sons[0], n.sons[0])
+      result = n
     else:
       GlobalError(n.info, "cannot map value from FFI " & typeToString(typ))
   of tyObject, tyTuple:
@@ -372,12 +385,24 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
     GlobalError(n.info, "cannot map value from FFI " & typeToString(typ))
 
 proc fficast*(x: PNode, destTyp: PType): PNode =
-  # we play safe here and allocate the max possible size:
-  let allocSize = max(packSize(x, x.typ), packSize(x, destTyp))
-  var a = alloc0(allocSize)
-  pack(x, x.typ, a)
-  result = unpack(a, destTyp, nil)
-  dealloc a
+  if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyPointer, 
+                                           tyProc, tyCString, tyString, 
+                                           tySequence}:
+    result = newNodeIT(x.kind, x.info, destTyp)
+    result.intVal = x.intVal
+  elif x.kind == nkNilLit:
+    result = newNodeIT(x.kind, x.info, destTyp)
+  else:
+    # we play safe here and allocate the max possible size:
+    let size = max(packSize(x, x.typ), packSize(x, destTyp))
+    var a = alloc0(size)
+    pack(x, x.typ, a)
+    # cast through a pointer needs a new inner object:
+    let y = if x.kind == nkRefTy: newNodeI(nkRefTy, x.info, 1)
+            else: x.copyTree
+    y.typ = x.typ
+    result = unpack(a, destTyp, y)
+    dealloc a
 
 proc callForeignFunction*(call: PNode): PNode =
   InternalAssert call.sons[0].kind == nkPtrLit
@@ -413,6 +438,6 @@ proc callForeignFunction*(call: PNode): PNode =
     result.info = call.info
 
   if retVal != nil: dealloc retVal
-  for i in countdown(call.len-2, 0):
-    call.sons[i+1] = unpack(args[i], typ.sons[i+1], call[i+1])
-    dealloc args[i]
+  for i in 1 .. call.len-1:
+    call.sons[i] = unpack(args[i-1], typ.sons[i], call[i])
+    dealloc args[i-1]

compiler/llstream.nim

@@ -103,17 +103,24 @@ proc continueLine(line: string, inTripleString: bool): bool {.inline.} =
       line[0] == ' ' or
       line.endsWith(LineContinuationOprs+AdditionalLineContinuationOprs)
 
-proc LLreadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int = 
-  var inTripleString = false
+proc countTriples(s: string): int =
+  var i = 0
+  while i < s.len:
+    if s[i] == '"' and s[i+1] == '"' and s[i+2] == '"':
+      inc result
+      inc i, 2
+    inc i
+
+proc LLreadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int =
   s.s = ""
   s.rd = 0
   var line = newStringOfCap(120)
+  var triples = 0
   while ReadLineFromStdin(if s.s.len == 0: ">>> " else: "... ", line): 
     add(s.s, line)
     add(s.s, "\n")
-    if line.contains("\"\"\""):
-      inTripleString = not inTripleString
-    if not continueLine(line, inTripleString): break
+    inc triples, countTriples(line)
+    if not continueLine(line, (triples and 1) == 1): break
   inc(s.lineOffset)
   result = min(bufLen, len(s.s) - s.rd)
   if result > 0: 

lib/system.nim

@@ -1066,86 +1066,87 @@ proc substr*(s: string, first, last: int): string {.
   ## is used instead: This means ``substr`` can also be used to `cut`:idx:
   ## or `limit`:idx: a string's length.
 
-proc zeroMem*(p: Pointer, size: int) {.importc, noDecl.}
-  ## overwrites the contents of the memory at ``p`` with the value 0.
-  ## Exactly ``size`` bytes will be overwritten. Like any procedure
-  ## dealing with raw memory this is *unsafe*.
+when not defined(nimrodVM):
+  proc zeroMem*(p: Pointer, size: int) {.importc, noDecl.}
+    ## overwrites the contents of the memory at ``p`` with the value 0.
+    ## Exactly ``size`` bytes will be overwritten. Like any procedure
+    ## dealing with raw memory this is *unsafe*.
 
-proc copyMem*(dest, source: Pointer, size: int) {.importc: "memcpy", noDecl.}
-  ## copies the contents from the memory at ``source`` to the memory
-  ## at ``dest``. Exactly ``size`` bytes will be copied. The memory
-  ## regions may not overlap. Like any procedure dealing with raw
-  ## memory this is *unsafe*.
+  proc copyMem*(dest, source: Pointer, size: int) {.importc: "memcpy", noDecl.}
+    ## copies the contents from the memory at ``source`` to the memory
+    ## at ``dest``. Exactly ``size`` bytes will be copied. The memory
+    ## regions may not overlap. Like any procedure dealing with raw
+    ## memory this is *unsafe*.
 
-proc moveMem*(dest, source: Pointer, size: int) {.importc: "memmove", noDecl.}
-  ## copies the contents from the memory at ``source`` to the memory
-  ## at ``dest``. Exactly ``size`` bytes will be copied. The memory
-  ## regions may overlap, ``moveMem`` handles this case appropriately
-  ## and is thus somewhat more safe than ``copyMem``. Like any procedure
-  ## dealing with raw memory this is still *unsafe*, though.
+  proc moveMem*(dest, source: Pointer, size: int) {.importc: "memmove", noDecl.}
+    ## copies the contents from the memory at ``source`` to the memory
+    ## at ``dest``. Exactly ``size`` bytes will be copied. The memory
+    ## regions may overlap, ``moveMem`` handles this case appropriately
+    ## and is thus somewhat more safe than ``copyMem``. Like any procedure
+    ## dealing with raw memory this is still *unsafe*, though.
 
-proc equalMem*(a, b: Pointer, size: int): bool {.
-  importc: "equalMem", noDecl, noSideEffect.}
-  ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will
-  ## be compared. If the blocks are equal, true is returned, false
-  ## otherwise. Like any procedure dealing with raw memory this is
-  ## *unsafe*.
+  proc equalMem*(a, b: Pointer, size: int): bool {.
+    importc: "equalMem", noDecl, noSideEffect.}
+    ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will
+    ## be compared. If the blocks are equal, true is returned, false
+    ## otherwise. Like any procedure dealing with raw memory this is
+    ## *unsafe*.
 
-proc alloc*(size: int): pointer {.noconv, rtl, tags: [].}
-  ## allocates a new memory block with at least ``size`` bytes. The
-  ## block has to be freed with ``realloc(block, 0)`` or
-  ## ``dealloc(block)``. The block is not initialized, so reading
-  ## from it before writing to it is undefined behaviour!
-  ## The allocated memory belongs to its allocating thread!
-  ## Use `allocShared` to allocate from a shared heap.
-proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].}
-  ## allocates a new memory block with at least ``size`` bytes. The
-  ## block has to be freed with ``realloc(block, 0)`` or
-  ## ``dealloc(block)``. The block is initialized with all bytes
-  ## containing zero, so it is somewhat safer than ``alloc``.
-  ## The allocated memory belongs to its allocating thread!
-  ## Use `allocShared0` to allocate from a shared heap.
-proc realloc*(p: Pointer, newsize: int): pointer {.noconv, rtl, tags: [].}
-  ## grows or shrinks a given memory block. If p is **nil** then a new
-  ## memory block is returned. In either way the block has at least
-  ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
-  ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to
-  ## be freed with ``dealloc``.
-  ## The allocated memory belongs to its allocating thread!
-  ## Use `reallocShared` to reallocate from a shared heap.
-proc dealloc*(p: Pointer) {.noconv, rtl, tags: [].}
-  ## frees the memory allocated with ``alloc``, ``alloc0`` or
-  ## ``realloc``. This procedure is dangerous! If one forgets to
-  ## free the memory a leak occurs; if one tries to access freed
-  ## memory (or just freeing it twice!) a core dump may happen
-  ## or other memory may be corrupted. 
-  ## The freed memory must belong to its allocating thread!
-  ## Use `deallocShared` to deallocate from a shared heap.
+  proc alloc*(size: int): pointer {.noconv, rtl, tags: [].}
+    ## allocates a new memory block with at least ``size`` bytes. The
+    ## block has to be freed with ``realloc(block, 0)`` or
+    ## ``dealloc(block)``. The block is not initialized, so reading
+    ## from it before writing to it is undefined behaviour!
+    ## The allocated memory belongs to its allocating thread!
+    ## Use `allocShared` to allocate from a shared heap.
+  proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].}
+    ## allocates a new memory block with at least ``size`` bytes. The
+    ## block has to be freed with ``realloc(block, 0)`` or
+    ## ``dealloc(block)``. The block is initialized with all bytes
+    ## containing zero, so it is somewhat safer than ``alloc``.
+    ## The allocated memory belongs to its allocating thread!
+    ## Use `allocShared0` to allocate from a shared heap.
+  proc realloc*(p: Pointer, newsize: int): pointer {.noconv, rtl, tags: [].}
+    ## grows or shrinks a given memory block. If p is **nil** then a new
+    ## memory block is returned. In either way the block has at least
+    ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
+    ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to
+    ## be freed with ``dealloc``.
+    ## The allocated memory belongs to its allocating thread!
+    ## Use `reallocShared` to reallocate from a shared heap.
+  proc dealloc*(p: Pointer) {.noconv, rtl, tags: [].}
+    ## frees the memory allocated with ``alloc``, ``alloc0`` or
+    ## ``realloc``. This procedure is dangerous! If one forgets to
+    ## free the memory a leak occurs; if one tries to access freed
+    ## memory (or just freeing it twice!) a core dump may happen
+    ## or other memory may be corrupted. 
+    ## The freed memory must belong to its allocating thread!
+    ## Use `deallocShared` to deallocate from a shared heap.
 
-proc allocShared*(size: int): pointer {.noconv, rtl.}
-  ## allocates a new memory block on the shared heap with at
-  ## least ``size`` bytes. The block has to be freed with
-  ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block
-  ## is not initialized, so reading from it before writing to it is 
-  ## undefined behaviour!
-proc allocShared0*(size: int): pointer {.noconv, rtl.}
-  ## allocates a new memory block on the shared heap with at 
-  ## least ``size`` bytes. The block has to be freed with
-  ## ``reallocShared(block, 0)`` or ``deallocShared(block)``.
-  ## The block is initialized with all bytes
-  ## containing zero, so it is somewhat safer than ``allocShared``.
-proc reallocShared*(p: Pointer, newsize: int): pointer {.noconv, rtl.}
-  ## grows or shrinks a given memory block on the heap. If p is **nil**
-  ## then a new memory block is returned. In either way the block has at least
-  ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
-  ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the
-  ## block has to be freed with ``deallocShared``.
-proc deallocShared*(p: Pointer) {.noconv, rtl.}
-  ## frees the memory allocated with ``allocShared``, ``allocShared0`` or
-  ## ``reallocShared``. This procedure is dangerous! If one forgets to
-  ## free the memory a leak occurs; if one tries to access freed
-  ## memory (or just freeing it twice!) a core dump may happen
-  ## or other memory may be corrupted.
+  proc allocShared*(size: int): pointer {.noconv, rtl.}
+    ## allocates a new memory block on the shared heap with at
+    ## least ``size`` bytes. The block has to be freed with
+    ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block
+    ## is not initialized, so reading from it before writing to it is 
+    ## undefined behaviour!
+  proc allocShared0*(size: int): pointer {.noconv, rtl.}
+    ## allocates a new memory block on the shared heap with at 
+    ## least ``size`` bytes. The block has to be freed with
+    ## ``reallocShared(block, 0)`` or ``deallocShared(block)``.
+    ## The block is initialized with all bytes
+    ## containing zero, so it is somewhat safer than ``allocShared``.
+  proc reallocShared*(p: Pointer, newsize: int): pointer {.noconv, rtl.}
+    ## grows or shrinks a given memory block on the heap. If p is **nil**
+    ## then a new memory block is returned. In either way the block has at least
+    ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
+    ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the
+    ## block has to be freed with ``deallocShared``.
+  proc deallocShared*(p: Pointer) {.noconv, rtl.}
+    ## frees the memory allocated with ``allocShared``, ``allocShared0`` or
+    ## ``reallocShared``. This procedure is dangerous! If one forgets to
+    ## free the memory a leak occurs; if one tries to access freed
+    ## memory (or just freeing it twice!) a core dump may happen
+    ## or other memory may be corrupted.
 
 proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.}
   ## swaps the values `a` and `b`. This is often more efficient than
@@ -1221,15 +1222,16 @@ const
 
 # GC interface:
 
-proc getOccupiedMem*(): int {.rtl.}
-  ## returns the number of bytes that are owned by the process and hold data.
+when not defined(nimrodVM):
+  proc getOccupiedMem*(): int {.rtl.}
+    ## returns the number of bytes that are owned by the process and hold data.
 
-proc getFreeMem*(): int {.rtl.}
-  ## returns the number of bytes that are owned by the process, but do not
-  ## hold any meaningful data.
+  proc getFreeMem*(): int {.rtl.}
+    ## returns the number of bytes that are owned by the process, but do not
+    ## hold any meaningful data.
 
-proc getTotalMem*(): int {.rtl.}
-  ## returns the number of bytes that are owned by the process.
+  proc getTotalMem*(): int {.rtl.}
+    ## returns the number of bytes that are owned by the process.
 
 
 iterator countdown*[T](a, b: T, step = 1): T {.inline.} =
@@ -1942,25 +1944,25 @@ when not defined(EcmaScript): #and not defined(NimrodVM):
 
   # -------------------------------------------------------------------------
 
-  proc allocCStringArray*(a: openArray[string]): cstringArray =
-    ## creates a NULL terminated cstringArray from `a`. The result has to
-    ## be freed with `deallocCStringArray` after it's not needed anymore.
-    result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring)))
-    for i in 0 .. a.high:
-      # XXX get rid of this string copy here:
-      var x = a[i]
-      result[i] = cast[cstring](alloc0(x.len+1))
-      copyMem(result[i], addr(x[0]), x.len)
-
-  proc deallocCStringArray*(a: cstringArray) =
-    ## frees a NULL terminated cstringArray.
-    var i = 0
-    while a[i] != nil:
-      dealloc(a[i])
-      inc(i)
-    dealloc(a)
-
   when not defined(NimrodVM):
+    proc allocCStringArray*(a: openArray[string]): cstringArray =
+      ## creates a NULL terminated cstringArray from `a`. The result has to
+      ## be freed with `deallocCStringArray` after it's not needed anymore.
+      result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring)))
+      for i in 0 .. a.high:
+        # XXX get rid of this string copy here:
+        var x = a[i]
+        result[i] = cast[cstring](alloc0(x.len+1))
+        copyMem(result[i], addr(x[0]), x.len)
+
+    proc deallocCStringArray*(a: cstringArray) =
+      ## frees a NULL terminated cstringArray.
+      var i = 0
+      while a[i] != nil:
+        dealloc(a[i])
+        inc(i)
+      dealloc(a)
+
     proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable.}
       ## atomic increment of `memLoc`. Returns the value after the operation.
     

todo.txt

@@ -2,9 +2,7 @@ version 0.9.2
 =============
 
 - FFI:
-  * make system.nim aware of nimffi
   * test libffi on windows
-  * test: SDL with the FFI
   * test: times.format with the FFI
 - fix closure bug finally
 - fix marshal bug
@@ -55,7 +53,9 @@ version 0.9.XX
 - object branch transitions can't work with the current 'reset'; add a 'reset'
   with an additional parameter --> re-evaluate this issue after constructors
   have been added
-- fix destructors; don't work yet when used as expression
+- fix destructors; don't work yet when used as expression; alternative for 
+  version 1: disallow expressions yielding a type with a destructor that are
+  not in a 'let/var' context  (p(a.openFile, b.openFile) makes no sense anyway)
 - document nimdoc properly finally
 - make 'clamp' a magic for the range stuff