bugfix: internal error in evalFieldAccess; parseutils.interpolatedFragments optimized; tstringinterp.nim now works

compiler/evals.nim

@@ -346,8 +346,13 @@ proc evalFieldAccess(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
   var field = n.sons[1].sym
   for i in countup(0, sonsLen(x) - 1): 
     var it = x.sons[i]
-    if it.kind != nkExprColonExpr: 
-      InternalError(it.info, "evalFieldAccess")
+    if it.kind != nkExprColonExpr:
+      # lookup per index:
+      result = x.sons[field.position]
+      if result.kind == nkExprColonExpr: result = result.sons[1]
+      if not aliasNeeded(result, flags): result = copyTree(result)
+      return
+      #InternalError(it.info, "evalFieldAccess")
     if it.sons[0].sym.name.id == field.name.id: 
       result = x.sons[i].sons[1]
       if not aliasNeeded(result, flags): result = copyTree(result)
@@ -752,29 +757,28 @@ proc evalRepr(c: PEvalContext, n: PNode): PNode =
 proc isEmpty(n: PNode): bool = 
   result = (n != nil) and (n.kind == nkEmpty)
 
-# The lexer marks multi-line strings as residing at the line where they are closed
-# This function returns the line where the string begins
-# Maybe the lexer should mark both the beginning and the end of expressions, then
-# this function could be removed
+# The lexer marks multi-line strings as residing at the line where they 
+# are closed. This function returns the line where the string begins
+# Maybe the lexer should mark both the beginning and the end of expressions,
+# then this function could be removed.
 proc stringStartingLine(s: PNode): int =
   var totalLines = 0
   for ln in splitLines(s.strVal): inc totalLines
-
   result = s.info.line - totalLines
 
 proc evalParseExpr(c: PEvalContext, n: Pnode): Pnode =
   var code = evalAux(c, n.sons[1], {})
-  var ast = parseString(code.getStrValue, code.info.toFilename, code.stringStartingLine)
-
+  var ast = parseString(code.getStrValue, code.info.toFilename,
+                        code.stringStartingLine)
   if sonsLen(ast) != 1:
     GlobalError(code.info, errExprExpected, "multiple statements")
-
   result = ast.sons[0]
   result.typ = newType(tyExpr, c.module)
 
 proc evalParseStmt(c: PEvalContext, n: Pnode): Pnode =
   var code = evalAux(c, n.sons[1], {})
-  result = parseString(code.getStrValue, code.info.toFilename, code.stringStartingLine)
+  result = parseString(code.getStrValue, code.info.toFilename,
+                       code.stringStartingLine)
   result.typ = newType(tyStmt, c.module)
 
 proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode = 
@@ -1086,7 +1090,7 @@ proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
     var a = copyTree(n)
     for i in countup(0, sonsLen(n) - 1): 
       var it = n.sons[i]
-      if it.kind == nkExprEqExpr:
+      if it.kind == nkExprColonExpr:
         result = evalAux(c, it.sons[1], flags)
         if isSpecial(result): return 
         a.sons[i].sons[1] = result

doc/manual.txt

@@ -3405,9 +3405,9 @@ exception in one thread terminates the whole *process*!
 Taint mode
 ==========
 
-The Nimrod compiler and standard library support a `taint mode`:idx:. 
-Input strings are declared with the `TaintedString`:idx: string type declared
-in the ``system`` module.
+The Nimrod compiler and most parts of the standard library support 
+a `taint mode`:idx:. Input strings are declared with the `TaintedString`:idx: 
+string type declared in the ``system`` module.
 
 If the taint mode is turned on (via the ``--taintMode:on`` command line 
 option) it is a distinct string type which helps to detect input
@@ -3423,4 +3423,3 @@ validation errors:
 If the taint mode is turned off, ``TaintedString`` is simply an alias for
 ``string``.
 
-

doc/tut1.txt

@@ -583,6 +583,16 @@ allow to silently throw away a return value:
   discard yes("May I ask a pointless question?")
 
 
+The return value can be ignored implicitely if the called proc/iterator has
+been declared with the ``discardable`` pragma: 
+
+.. code-block:: nimrod
+  proc p(x, y: int): int {.discardable.} = 
+    return x + y
+    
+  p(3, 4) # now valid
+
+
 Named arguments
 ---------------
 

lib/core/macros.nim

@@ -220,13 +220,13 @@ proc toYaml*(n: PNimrodNode): string {.magic: "AstToYaml".}
   ## Provides more detailed, potentially harder to digest information
   ## than `toLisp`
 
-proc parseExpr*(s: string) : expr {.magic: "ParseExprToAst".}
-  ## Compiles the passed string to its AST representation
-  ## Expects a single expression
+proc parseExpr*(s: string): expr {.magic: "ParseExprToAst".}
+  ## Compiles the passed string to its AST representation.
+  ## Expects a single expression.
 
-proc parseStmt*(s: string) : stmt {.magic: "ParseStmtToAst".}
-  ## Compiles the passed string to its AST representation
-  ## Expects one or more statements
+proc parseStmt*(s: string): stmt {.magic: "ParseStmtToAst".}
+  ## Compiles the passed string to its AST representation.
+  ## Expects one or more statements.
 
 proc getAst*(macroOrTemplate: expr): expr {.magic: "ExpandMacroToAst".}
   ## Obtains the AST nodes returned from a macro or template invocation.

lib/pure/parsecfg.nim

@@ -70,7 +70,7 @@ type
 # implementation
 
 const 
-  SymChars: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF'} 
+  SymChars: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.'} 
   
 proc rawGetTok(c: var TCfgParser, tok: var TToken)
 

lib/pure/parseutils.nim

@@ -266,94 +266,76 @@ proc parseFloat*(s: string, number: var float, start = 0): int {.
   result = parseBiggestFloat(s, bf, start)
   number = bf
   
-proc isEscaped*(s: string, pos: int) : bool =
-  assert pos >= 0 and pos < s.len
-
-  var
-    backslashes = 0
-    j = pos - 1
-
-  while j >= 0:
-    if s[j] == '\\':
-      inc backslashes
-      dec j
-    else:
-      break
-
-  return backslashes mod 2 != 0
-
 type
-  TInterpolatedKind* = enum
-    ikString, ikExpr
-
-  TInterpStrFragment* = tuple[kind: TInterpolatedKind, value: string]
-
-iterator interpolatedFragments*(s: string): TInterpStrFragment =
-  var 
-    i = 0
-    tokenStart = 0
-
-  proc token(kind: TInterpolatedKind, value: string): TInterpStrFragment =
-    result.kind = kind
-    result.value = value
-
-  while i < s.len:
-    # The $ sign marks the start of an interpolation.
-    #
-    # It's followed either by a varialbe name or an opening bracket 
-    # (so it should be before the end of the string)
-    # if the dollar sign is escaped, don't trigger interpolation
-    if s[i] == '$' and i < (s.len - 1) and not isEscaped(s, i):
-      # Interpolation starts here.
-      # Return any string that we've ran over so far.
-      if i != tokenStart:
-        yield token(ikString, s[tokenStart..i-1])
-
-      var next = s[i+1]
-      if next == '{':
-        # Complex expression: ${foo(bar) in {1..100}}
-        # Find closing braket, while respecting any nested brackets
-        inc i
-        tokenStart = i + 1
-
-        var
-          brackets = {'{', '}'}
-          nestingCount = 1
-        
-        while i < s.len:
-          inc i, skipUntil(s, brackets, i+1) + 1
-          
-          if not isEscaped(s, i):
-            if s[i] == '}':
-              dec nestingCount
-              if nestingCount == 0: break
-            else:
-              inc nestingCount
-
-        yield token(ikExpr, s[tokenStart..(i-1)])
-
-        tokenStart = i + 1
-        
+  TInterpolatedKind* = enum  ## describes for `interpolatedFragments`
+                             ## which part of the interpolated string is
+                             ## yielded; for example in "str$var${expr}"
+    ikStr,                   ## ``str`` part of the interpolated string 
+    ikVar,                   ## ``var`` part of the interpolated string
+    ikExpr                   ## ``expr`` part of the interpolated string
+
+iterator interpolatedFragments*(s: string): tuple[kind: TInterpolatedKind,
+  value: string] =
+  ## Tokenizes the string `s` into substrings for interpolation purposes.
+  ##
+  ## Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   for k, v in interpolatedFragments("  $this is ${an  example}  "):
+  ##     echo "(", k, ", \"", v, "\")"
+  ##
+  ## Results in:
+  ##
+  ## .. code-block:: nimrod
+  ##   (ikString, "  ")
+  ##   (ikExpr, "this")
+  ##   (ikString, " is ")
+  ##   (ikExpr, "an  example")
+  ##   (ikString, "  ")
+  var i = 0
+  var kind: TInterpolatedKind
+  while true:
+    var j = i
+    if s[j] == '$' and s[j+1] != '$':
+      if s[j+1] == '{':
+        inc j, 2
+        var nesting = 0
+        while true:
+          case s[j]
+          of '{': inc nesting
+          of '}':
+            if nesting == 0: 
+              inc j
+              break
+            dec nesting
+          of '\0':
+            raise newException(EInvalidValue, 
+              "Expected closing '}': " & s[i..s.len])
+          else: nil
+          inc j
+        inc i, 2 # skip ${
+        kind = ikExpr
+      elif s[j+1] in IdentStartChars:
+        inc j, 2
+        while s[j] in IdentChars: inc(j)
+        inc i # skip $
+        kind = ikVar
       else:
-        tokenStart = i + 1
-        var identifier = parseIdent(s, i+1)
-        
-        if identifier.len >  0:
-          inc i, identifier.len
+        raise newException(EInvalidValue, 
+          "Unable to parse a varible name at " & s[i..s.len])
+    else:
+      while j < s.len and (s[j] != '$' or s[j+1] == '$'): inc j
+      kind = ikStr
+    if j > i:
+      # do not copy the trailing } for ikExpr:
+      yield (kind, substr(s, i, j-1-ord(kind == ikExpr)))
+    else:
+      break
+    i = j
 
-          yield token(ikExpr, s[tokenStart..i])
+when isMainModule:
+  for k, v in interpolatedFragments("$test{}  $this is ${an{  example}}  "):
+    echo "(", k, ", \"", v, "\")"
 
-          tokenStart = i + 1
-
-        else:
-          raise newException(EInvalidValue, "Unable to parse a varible name at " & s[i..s.len])
-       
-    inc i
-  #end while
-
-  # We've reached the end of the string without finding a new interpolation.
-  # Return the last fragment at string.
-  if i != tokenStart:
-    yield token(ikString, s[tokenStart..i])
 
 {.pop.}

lib/pure/strutils.nim

@@ -640,8 +640,7 @@ proc join*(a: openArray[string], sep: string): string {.
   if len(a) > 0:
     var L = sep.len * (a.len-1)
     for i in 0..high(a): inc(L, a[i].len)
-    result = newString(L)
-    setLen(result, 0)
+    result = newStringOfCap(L)
     add(result, a[0])
     for i in 1..high(a):
       add(result, sep)
@@ -655,8 +654,7 @@ proc join*(a: openArray[string]): string {.
   if len(a) > 0:
     var L = 0
     for i in 0..high(a): inc(L, a[i].len)
-    result = newString(L)
-    setLen(result, 0)
+    result = newStringOfCap(L)
     for i in 0..high(a): add(result, a[i])
   else:
     result = ""
@@ -867,9 +865,9 @@ proc validIdentifier*(s: string): bool {.noSideEffect,
 
 proc editDistance*(a, b: string): int {.noSideEffect,
   rtl, extern: "nsuEditDistance".} =
-  ## returns the edit distance between `a` and `b`. This uses the Levenshtein
-  ## distance algorithm with only a linear memory overhead. This implementation
-  ## is highly optimized!
+  ## returns the edit distance between `a` and `b`. This uses the 
+  ## `Levenshtein`:idx: distance algorithm with only a linear memory overhead.
+  ## This implementation is highly optimized!
   var len1 = a.len
   var len2 = b.len
   if len1 > len2:

tests/accept/run/tstringinterp.nim

@@ -5,20 +5,17 @@ discard """
 
 import macros, parseutils, strutils
 
-proc concat(strings: openarray[string]) : string =
+proc concat(strings: openarray[string]): string =
   result = newString(0)
   for s in items(strings): result.add(s)
 
-# This will run though the intee
 template ProcessInterpolations(e: expr) =
-  var 
-    s = e[1].strVal
-    
+  var s = e[1].strVal
   for f in interpolatedFragments(s):
-    if f.kind  == ikString:
+    if f.kind == ikStr:
       addString(f.value)
     else:
-      addExpr(f.value)
+      addExpr(newCall("$", parseExpr(f.value)))
 
 macro formatStyleInterpolation(e: expr): expr =
   var 
@@ -41,7 +38,7 @@ macro formatStyleInterpolation(e: expr): expr =
   result[2] = arrayNode
 
 macro concatStyleInterpolation(e: expr): expr =
-  var args : seq[PNimrodNode]
+  var args: seq[PNimrodNode]
   newSeq(args, 0)
 
   proc addString(s: string)  = args.add(newStrLitNode(s))
@@ -64,8 +61,8 @@ var
   c = 34
 
 var
-  s1 = concatStyleInterpolation"Hello ${alice}, ${sum (a, b, c)}}"
-  s2 = formatStyleInterpolation"Hello ${bob}, ${sum (alice.len, bob.len, 2)}"
+  s1 = concatStyleInterpolation"Hello ${alice}, ${sum(a, b, c)}"
+  s2 = formatStyleInterpolation"Hello ${bob}, ${sum(alice.len, bob.len, 2)}"
 
 write(stdout, s1 & " | " & s2)
 

todo.txt

@@ -73,7 +73,6 @@ version 0.9.XX
 Library
 -------
 
-- proper URL-parser
 - wrappers for poppler; libharu
 - radix tree for strings; maybe suffix tree
 - locale support

web/news.txt

@@ -17,6 +17,7 @@ Bugfixes
   the end of file for text files that do not end with a newline.
 - Bugfix c2nim, c2pas: the ``--out`` option has never worked properly.
 - Bugfix: forwarding of generic procs never worked.
+- Some more bugfixes for macros and compile-time evaluation.
 
 
 Changes affecting backwards compatibility
@@ -80,6 +81,9 @@ Library Additions
 - Added ``system.running`` for threads.
 - Added ``xmltree.innerText``.
 - Added ``os.isAbsolute``.
+- Added ``parseutils.interpolatedFragments``.
+- Added ``macros.toLisp``, ``macros.toYaml``, ``macros.parseExpr``,
+  ``macros.parseStmt``, ``macros.getAst``.
 - Added ``locks`` core module for more flexible locking support.
 - Added ``irc`` module.