documented hygienic templates; made tests green; fixed system.clamp

compiler/semtempl.nim

@@ -52,7 +52,9 @@ proc symChoice(c: PContext, n: PNode, s: PSym): PNode =
     if i > 1: break
   if i <= 1:
     # XXX this makes more sense but breaks bootstrapping for now:
-    # and s.kind notin routineKinds:
+    # (s.kind notin routineKinds or s.magic != mNone):
+    # for some reason 'nextTry' is copied and considered as a candidate in
+    # tables.nim
     result = newSymNode(s, n.info)
     markUsed(n, s)
   else:
@@ -68,12 +70,19 @@ proc symChoice(c: PContext, n: PNode, s: PSym): PNode =
 proc semBindStmt(c: PContext, n: PNode, toBind: var TIntSet): PNode =
   for i in 0 .. < n.len:
     var a = n.sons[i]
-    # If 'a' is an overloaded symbol, we use the first symbol as a 'witness'
-    # and use the fact that subsequent lookups will yield the same symbol!
-    # This is currently the case due to the hash table's implementation...
+    # If 'a' is an overloaded symbol, we used to use the first symbol
+    # as a 'witness' and use the fact that subsequent lookups will yield
+    # the same symbol!
+    # This is however not true anymore for hygienic templates as semantic
+    # processing for them changes the symbol table...
     let s = QualifiedLookUp(c, a)
     if s != nil:
-      toBind.incl(s.id)
+      # we need to mark all symbols:
+      let sc = symChoice(c, n, s)
+      if sc.kind == nkSym:
+        toBind.incl(sc.sym.id)
+      else:
+        for x in items(sc): toBind.incl(x.sym.id)
     else:
       illFormedAst(a)
   result = newNodeI(nkEmpty, n.info)

doc/manual.txt

@@ -2967,8 +2967,8 @@ In templates identifiers can be constructed with the backticks notation:
 
   template typedef(name: expr, typ: typeDesc) {.immediate.} = 
     type
-      `T name`* = typ
-      `P name`* = ref `T name`
+      `T name`* {.inject.} = typ
+      `P name`* {.inject.} = ref `T name`
       
   typedef(myint, int)
   var x: PMyInt
@@ -2982,7 +2982,7 @@ Lookup rules for template parameters
 
 A parameter ``p`` in a template is even substituted in the expression ``x.p``.
 Thus template arguments can be used as field names and a global symbol can be
-covered by the same argument name even when fully qualified:
+shadowed by the same argument name even when fully qualified:
 
 .. code-block:: nimrod
   # module 'm'
@@ -3017,6 +3017,61 @@ But the global symbol can properly be captured by a ``bind`` statement:
   tstLev(levA)
   # produces: 'levA levB'
 
+
+Hygiene in templates
+~~~~~~~~~~~~~~~~~~~~
+
+Per default templates are `hygienic`:idx:\: Local identifiers declared in a
+template cannot be accessed in the instantiation context:
+
+.. code-block:: nimrod
+  
+  template newException*(exceptn: typeDesc, message: string): expr =
+    var
+      e: ref exceptn  # e is implicitely gensym'ed here
+    new(e)
+    e.msg = message
+    e
+    
+  # so this works:
+  let e = "message"
+  raise newException(EIO, e)
+
+
+Whether a symbol that is declared in a template is exposed to the instantiation
+scope is controlled by the `inject`:idx: and `gensym`:idx: pragmas: gensym'ed
+symbols are not exposed but inject'ed are.
+
+The default for symbols of entity ``type``, ``var``, ``let`` and ``const``
+is ``gensym`` and for ``proc``, ``iterator``, ``converter``, ``template``,
+``macro`` is ``inject``. However, if the name of the entity is passed as a 
+template parameter, it is an inject'ed symbol:
+
+.. code-block:: nimrod
+  template withFile(f, fn, mode: expr, actions: stmt): stmt {.immediate.} =
+    block:
+      var f: TFile  # since 'f' is a template param, it's injected implicitely
+      ...
+      
+  withFile(txt, "ttempl3.txt", fmWrite):
+    txt.writeln("line 1")
+    txt.writeln("line 2")
+
+
+The ``inject`` and ``gensym`` pragmas are second class annotations; they have
+no semantics outside of a template definition and cannot be abstracted over:
+
+.. code-block:: nimrod
+  {.pragma myInject: inject.}
+  
+  template t() =
+    var x {.myInject.}: int # does NOT work
+
+
+To get rid of hygiene in templates, one can use the `dirty`:idx: pragma for
+a template. ``inject`` and ``gensym`` have no effect in ``dirty`` templates.
+
+
 
 Macros
 ------

doc/nimrodc.txt

@@ -487,7 +487,7 @@ objects as `shallow`:idx:\:
 .. code-block:: Nimrod
   var s = "abc"
   shallow(s) # mark 's' as shallow string
-  var x = s  # now might does not copy the string!
+  var x = s  # now might not copy the string!
   
 Usage of ``shallow`` is always safe once you know the string won't be modified
 anymore, similar to Ruby's `freeze`:idx:.

lib/pure/collections/lists.nim

@@ -11,6 +11,9 @@
 ## to do so, the 'next' and 'prev' pointers are not hidden from you and can
 ## be manipulated directly for efficiency.
 
+when not defined(nimhygiene):
+  {.pragma: dirty.}
+
 type
   TDoublyLinkedNode* {.pure, 
       final.}[T] = object ## a node a doubly linked list consists of
@@ -62,13 +65,13 @@ proc newSinglyLinkedNode*[T](value: T): PSinglyLinkedNode[T] =
   new(result)
   result.value = value
 
-template itemsListImpl() =
+template itemsListImpl() {.dirty.} =
   var it = L.head
   while it != nil:
     yield it.value
     it = it.next
 
-template itemsRingImpl() =
+template itemsRingImpl() {.dirty.} =
   var it = L.head
   if it != nil:
     while true:
@@ -76,14 +79,14 @@ template itemsRingImpl() =
       it = it.next
       if it == L.head: break
 
-template nodesListImpl() =
+template nodesListImpl() {.dirty.} =
   var it = L.head
   while it != nil:
     var nxt = it.next
     yield it
     it = nxt
 
-template nodesRingImpl() =
+template nodesRingImpl() {.dirty.} =
   var it = L.head
   if it != nil:
     while true:
@@ -92,7 +95,7 @@ template nodesRingImpl() =
       it = nxt
       if it == L.head: break
 
-template findImpl() =
+template findImpl() {.dirty.} =
   for x in nodes(L):
     if x.value == value: return x
 
@@ -132,7 +135,7 @@ iterator nodes*[T](L: TDoublyLinkedRing[T]): PDoublyLinkedNode[T] =
   ## list during traversal is supported.
   nodesRingImpl()
 
-template dollarImpl() =
+template dollarImpl() {.dirty.} =
   result = "["
   for x in nodes(L):
     if result.len > 1: result.add(", ")

lib/pure/collections/sequtils.nim

@@ -15,6 +15,9 @@
 ## **Note**: This interface will change as soon as the compiler supports
 ## closures and proper coroutines.
 
+when not defined(nimhygiene):
+  {.pragma: dirty.}
+
 proc concat*[T](seqs: varargs[seq[T]]): seq[T] =
   ## Takes several sequences' items and returns them inside of one sequence.
   var L = 0
@@ -50,7 +53,7 @@ proc filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): seq[T] =
   ## Returns all items in a sequence that fulfilled the predicate.
   accumulateResult(filter(seq1, pred))
 
-template filterIt*(seq1, pred: expr): expr {.immediate.} =
+template filterIt*(seq1, pred: expr): expr {.immediate, dirty.} =
   ## Finds a specific item in a sequence as long as the 
   ## predicate returns true. The predicate needs to be an expression
   ## containing ``it``: ``filterIt("abcxyz", it == 'x')``.

lib/pure/collections/sets.nim

@@ -16,6 +16,8 @@ import
   os, hashes, math
 
 {.pragma: myShallow.}
+when not defined(nimhygiene):
+  {.pragma: dirty.}
 
 type
   TSlotEnum = enum seEmpty, seFilled, seDeleted
@@ -48,7 +50,7 @@ proc mustRehash(length, counter: int): bool {.inline.} =
 proc nextTry(h, maxHash: THash): THash {.inline.} =
   result = ((5 * h) + 1) and maxHash
 
-template rawGetImpl() =
+template rawGetImpl() {.dirty.} =
   var h: THash = hash(key) and high(s.data) # start with real hash value
   while s.data[h].slot != seEmpty:
     if s.data[h].key == key and s.data[h].slot == seFilled:
@@ -56,7 +58,7 @@ template rawGetImpl() =
     h = nextTry(h, high(s.data))
   result = -1
 
-template rawInsertImpl() =
+template rawInsertImpl() {.dirty.} =
   var h: THash = hash(key) and high(data)
   while data[h].slot == seFilled:
     h = nextTry(h, high(data))
@@ -81,14 +83,14 @@ proc Enlarge[A](s: var TSet[A]) =
     if s.data[i].slot == seFilled: RawInsert(s, n, s.data[i].key)
   swap(s.data, n)
 
-template inclImpl() =
+template inclImpl() {.dirty.} =
   var index = RawGet(s, key)
   if index < 0:
     if mustRehash(len(s.data), s.counter): Enlarge(s)
     RawInsert(s, s.data, key)
     inc(s.counter)
 
-template containsOrInclImpl() =
+template containsOrInclImpl() {.dirty.} =
   var index = RawGet(s, key)
   if index >= 0:
     result = true
@@ -125,7 +127,7 @@ proc toSet*[A](keys: openarray[A]): TSet[A] =
   result = initSet[A](nextPowerOfTwo(keys.len+10))
   for key in items(keys): result.incl(key)
 
-template dollarImpl(): stmt =
+template dollarImpl(): stmt {.dirty.} =
   result = "{"
   for key in items(s):
     if result.len > 1: result.add(", ")
@@ -155,7 +157,7 @@ proc card*[A](s: TOrderedSet[A]): int {.inline.} =
   ## alias for `len`.
   result = s.counter
 
-template forAllOrderedPairs(yieldStmt: stmt) =
+template forAllOrderedPairs(yieldStmt: stmt) {.dirty.} =
   var h = s.first
   while h >= 0:
     var nxt = s.data[h].next

lib/pure/collections/tables.nim

@@ -26,6 +26,9 @@ type
     data: TKeyValuePairSeq[A, B]
     counter: int
 
+when not defined(nimhygiene):
+  {.pragma: dirty.}
+
 proc len*[A, B](t: TTable[A, B]): int =
   ## returns the number of keys in `t`.
   result = t.counter
@@ -66,7 +69,7 @@ proc mustRehash(length, counter: int): bool {.inline.} =
 proc nextTry(h, maxHash: THash): THash {.inline.} =
   result = ((5 * h) + 1) and maxHash
 
-template rawGetImpl() =
+template rawGetImpl() {.dirty.} =
   var h: THash = hash(key) and high(t.data) # start with real hash value
   while t.data[h].slot != seEmpty:
     if t.data[h].key == key and t.data[h].slot == seFilled:
@@ -74,7 +77,7 @@ template rawGetImpl() =
     h = nextTry(h, high(t.data))
   result = -1
 
-template rawInsertImpl() =
+template rawInsertImpl() {.dirty.} =
   var h: THash = hash(key) and high(data)
   while data[h].slot == seFilled:
     h = nextTry(h, high(data))
@@ -115,12 +118,12 @@ proc Enlarge[A, B](t: var TTable[A, B]) =
     if t.data[i].slot == seFilled: RawInsert(t, n, t.data[i].key, t.data[i].val)
   swap(t.data, n)
 
-template AddImpl() =
+template AddImpl() {.dirty.} =
   if mustRehash(len(t.data), t.counter): Enlarge(t)
   RawInsert(t, t.data, key, val)
   inc(t.counter)
 
-template PutImpl() =
+template PutImpl() {.dirty.} =
   var index = RawGet(t, key)
   if index >= 0:
     t.data[index].val = val
@@ -129,7 +132,7 @@ template PutImpl() =
 
 when false:
   # not yet used:
-  template HasKeyOrPutImpl() =
+  template HasKeyOrPutImpl() {.dirty.} =
     var index = RawGet(t, key)
     if index >= 0:
       t.data[index].val = val
@@ -168,7 +171,7 @@ proc toTable*[A, B](pairs: openarray[tuple[key: A,
   result = initTable[A, B](nextPowerOfTwo(pairs.len+10))
   for key, val in items(pairs): result[key] = val
 
-template dollarImpl(): stmt =
+template dollarImpl(): stmt {.dirty.} =
   if t.len == 0:
     result = "{:}"
   else:
@@ -199,7 +202,7 @@ proc len*[A, B](t: TOrderedTable[A, B]): int {.inline.} =
   ## returns the number of keys in `t`.
   result = t.counter
 
-template forAllOrderedPairs(yieldStmt: stmt) =
+template forAllOrderedPairs(yieldStmt: stmt) {.dirty.} =
   var h = t.first
   while h >= 0:
     var nxt = t.data[h].next

lib/pure/unittest.nim

@@ -31,19 +31,19 @@ var
   
   checkpoints: seq[string] = @[]
 
-template TestSetupIMPL*: stmt = nil
-template TestTeardownIMPL*: stmt = nil
+template TestSetupIMPL*: stmt {.dirty.} = nil
+template TestTeardownIMPL*: stmt {.dirty.} = nil
 
 proc shouldRun(testName: string): bool =
   result = true
 
-template suite*(name: expr, body: stmt): stmt =
+template suite*(name: expr, body: stmt): stmt {.dirty.} =
   block:
-    template setup*(setupBody: stmt): stmt =
-      template TestSetupIMPL: stmt = setupBody
+    template setup*(setupBody: stmt): stmt {.dirty.} =
+      template TestSetupIMPL: stmt {.dirty.} = setupBody
 
-    template teardown*(teardownBody: stmt): stmt =
-      template TestTeardownIMPL: stmt = teardownBody
+    template teardown*(teardownBody: stmt): stmt {.dirty.} =
+      template TestTeardownIMPL: stmt {.dirty.} = teardownBody
 
     body
 
@@ -59,12 +59,12 @@ proc testDone(name: string, s: TTestStatus) =
     else:
       echo "[", $s, "] ", name, "\n"
   
-template test*(name: expr, body: stmt): stmt =
+template test*(name: expr, body: stmt): stmt {.dirty.} =
   bind shouldRun, checkpoints, testDone
 
   if shouldRun(name):
     checkpoints = @[]
-    var TestStatusIMPL = OK
+    var TestStatusIMPL {.inject.} = OK
     
     try:
       TestSetupIMPL()
@@ -146,13 +146,14 @@ macro check*(conditions: stmt): stmt =
     var ast = conditions.treeRepr
     error conditions.lineinfo & ": Malformed check statement:\n" & ast
 
-template require*(conditions: stmt): stmt =
+template require*(conditions: stmt): stmt {.dirty.} =
   block:
-    const AbortOnError = true    
+    const AbortOnError {.inject.} = true
     check conditions
 
 macro expect*(exp: stmt): stmt =
-  template expectBody(errorTypes, lineInfoLit: expr, body: stmt): PNimrodNode =
+  template expectBody(errorTypes, lineInfoLit: expr,
+                      body: stmt): PNimrodNode {.dirty.} =
     try:
       body
       checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.")

lib/system.nim

@@ -1259,8 +1259,8 @@ proc max*[T](x: varargs[T]): T =
 
 proc clamp*[T](x, a, b: T): T =
   ## limits the value ``x`` within the interval [a, b] 
-  if x > a: return a
-  if x < b: return b
+  if x < a: return a
+  if x > b: return b
   return x
 
 iterator items*[T](a: openarray[T]): T {.inline.} =
@@ -1657,9 +1657,9 @@ proc echo*[T](x: varargs[T, `$`]) {.magic: "Echo", noSideEffect.}
   ## side effects, so that it can be used for debugging routines marked as
   ## ``noSideEffect``.
 
-template newException*(exceptn: typeDesc, message: string): expr = 
+template newException*(exceptn: typeDesc, message: string): expr =
   ## creates an exception object of type ``exceptn`` and sets its ``msg`` field
-  ## to `message`. Returns the new exception object. 
+  ## to `message`. Returns the new exception object.
   # block: # open a new scope
   var
     e: ref exceptn

tests/run/tpegs.nim

@@ -866,7 +866,7 @@ template `=~`*(s: string, pattern: TPeg): expr =
   ##     echo("syntax error")
   ##  
   when not definedInScope(matches):
-    var matches: array[0..maxSubpatterns-1, string]
+    var matches {.inject.}: array[0..maxSubpatterns-1, string]
   match(s, pattern, matches)
 
 # ------------------------- more string handling ------------------------------

todo.txt

@@ -1,9 +1,8 @@
 version 0.9.0
 =============
 
-- make templates hygienic by default: 'gensym', 'inject' pragmas;
-  document 'gensym', 'inject' and 'dirty'
 - make 'bind' default for templates and introduce 'mixin'
+- implement 'bind' for macros
 - use ``\`` for comment continuations
 - ``final`` should be the default for objects
 - implement "closure tuple consists of a single 'ref'" optimization