From 14968fba46a0c1128f38859b339c7384f9f96cd4 Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 26 Sep 2011 00:24:06 +0200 Subject: [PATCH] bugfix: internal error in evalFieldAccess; parseutils.interpolatedFragments optimized; tstringinterp.nim now works --- compiler/evals.nim | 28 +++--- doc/manual.txt | 7 +- doc/tut1.txt | 10 ++ lib/core/macros.nim | 12 +-- lib/pure/parsecfg.nim | 2 +- lib/pure/parseutils.nim | 152 +++++++++++++---------------- lib/pure/strutils.nim | 12 +-- tests/accept/run/tstringinterp.nim | 17 ++-- todo.txt | 1 - web/news.txt | 4 + 10 files changed, 119 insertions(+), 126 deletions(-) diff --git a/compiler/evals.nim b/compiler/evals.nim index 65c64f4d3f..a61d687ef5 100755 --- a/compiler/evals.nim +++ b/compiler/evals.nim @@ -346,8 +346,13 @@ proc evalFieldAccess(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode = var field = n.sons[1].sym for i in countup(0, sonsLen(x) - 1): var it = x.sons[i] - if it.kind != nkExprColonExpr: - InternalError(it.info, "evalFieldAccess") + if it.kind != nkExprColonExpr: + # lookup per index: + result = x.sons[field.position] + if result.kind == nkExprColonExpr: result = result.sons[1] + if not aliasNeeded(result, flags): result = copyTree(result) + return + #InternalError(it.info, "evalFieldAccess") if it.sons[0].sym.name.id == field.name.id: result = x.sons[i].sons[1] if not aliasNeeded(result, flags): result = copyTree(result) @@ -752,29 +757,28 @@ proc evalRepr(c: PEvalContext, n: PNode): PNode = proc isEmpty(n: PNode): bool = result = (n != nil) and (n.kind == nkEmpty) -# The lexer marks multi-line strings as residing at the line where they are closed -# This function returns the line where the string begins -# Maybe the lexer should mark both the beginning and the end of expressions, then -# this function could be removed +# The lexer marks multi-line strings as residing at the line where they +# are closed. This function returns the line where the string begins +# Maybe the lexer should mark both the beginning and the end of expressions, +# then this function could be removed. proc stringStartingLine(s: PNode): int = var totalLines = 0 for ln in splitLines(s.strVal): inc totalLines - result = s.info.line - totalLines proc evalParseExpr(c: PEvalContext, n: Pnode): Pnode = var code = evalAux(c, n.sons[1], {}) - var ast = parseString(code.getStrValue, code.info.toFilename, code.stringStartingLine) - + var ast = parseString(code.getStrValue, code.info.toFilename, + code.stringStartingLine) if sonsLen(ast) != 1: GlobalError(code.info, errExprExpected, "multiple statements") - result = ast.sons[0] result.typ = newType(tyExpr, c.module) proc evalParseStmt(c: PEvalContext, n: Pnode): Pnode = var code = evalAux(c, n.sons[1], {}) - result = parseString(code.getStrValue, code.info.toFilename, code.stringStartingLine) + result = parseString(code.getStrValue, code.info.toFilename, + code.stringStartingLine) result.typ = newType(tyStmt, c.module) proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode = @@ -1086,7 +1090,7 @@ proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode = var a = copyTree(n) for i in countup(0, sonsLen(n) - 1): var it = n.sons[i] - if it.kind == nkExprEqExpr: + if it.kind == nkExprColonExpr: result = evalAux(c, it.sons[1], flags) if isSpecial(result): return a.sons[i].sons[1] = result diff --git a/doc/manual.txt b/doc/manual.txt index 03e763190e..c088e7d831 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -3405,9 +3405,9 @@ exception in one thread terminates the whole *process*! Taint mode ========== -The Nimrod compiler and standard library support a `taint mode`:idx:. -Input strings are declared with the `TaintedString`:idx: string type declared -in the ``system`` module. +The Nimrod compiler and most parts of the standard library support +a `taint mode`:idx:. Input strings are declared with the `TaintedString`:idx: +string type declared in the ``system`` module. If the taint mode is turned on (via the ``--taintMode:on`` command line option) it is a distinct string type which helps to detect input @@ -3423,4 +3423,3 @@ validation errors: If the taint mode is turned off, ``TaintedString`` is simply an alias for ``string``. - diff --git a/doc/tut1.txt b/doc/tut1.txt index fb4bbd19ce..54268b372c 100755 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -583,6 +583,16 @@ allow to silently throw away a return value: discard yes("May I ask a pointless question?") +The return value can be ignored implicitely if the called proc/iterator has +been declared with the ``discardable`` pragma: + +.. code-block:: nimrod + proc p(x, y: int): int {.discardable.} = + return x + y + + p(3, 4) # now valid + + Named arguments --------------- diff --git a/lib/core/macros.nim b/lib/core/macros.nim index e362922c8f..c5afcdf174 100755 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -220,13 +220,13 @@ proc toYaml*(n: PNimrodNode): string {.magic: "AstToYaml".} ## Provides more detailed, potentially harder to digest information ## than `toLisp` -proc parseExpr*(s: string) : expr {.magic: "ParseExprToAst".} - ## Compiles the passed string to its AST representation - ## Expects a single expression +proc parseExpr*(s: string): expr {.magic: "ParseExprToAst".} + ## Compiles the passed string to its AST representation. + ## Expects a single expression. -proc parseStmt*(s: string) : stmt {.magic: "ParseStmtToAst".} - ## Compiles the passed string to its AST representation - ## Expects one or more statements +proc parseStmt*(s: string): stmt {.magic: "ParseStmtToAst".} + ## Compiles the passed string to its AST representation. + ## Expects one or more statements. proc getAst*(macroOrTemplate: expr): expr {.magic: "ExpandMacroToAst".} ## Obtains the AST nodes returned from a macro or template invocation. diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 3e85a9ee67..df56fdb840 100755 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -70,7 +70,7 @@ type # implementation const - SymChars: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF'} + SymChars: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.'} proc rawGetTok(c: var TCfgParser, tok: var TToken) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 1f4e54b96b..001872db2d 100755 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -266,94 +266,76 @@ proc parseFloat*(s: string, number: var float, start = 0): int {. result = parseBiggestFloat(s, bf, start) number = bf -proc isEscaped*(s: string, pos: int) : bool = - assert pos >= 0 and pos < s.len - - var - backslashes = 0 - j = pos - 1 - - while j >= 0: - if s[j] == '\\': - inc backslashes - dec j - else: - break - - return backslashes mod 2 != 0 - type - TInterpolatedKind* = enum - ikString, ikExpr - - TInterpStrFragment* = tuple[kind: TInterpolatedKind, value: string] - -iterator interpolatedFragments*(s: string): TInterpStrFragment = - var - i = 0 - tokenStart = 0 - - proc token(kind: TInterpolatedKind, value: string): TInterpStrFragment = - result.kind = kind - result.value = value - - while i < s.len: - # The $ sign marks the start of an interpolation. - # - # It's followed either by a varialbe name or an opening bracket - # (so it should be before the end of the string) - # if the dollar sign is escaped, don't trigger interpolation - if s[i] == '$' and i < (s.len - 1) and not isEscaped(s, i): - # Interpolation starts here. - # Return any string that we've ran over so far. - if i != tokenStart: - yield token(ikString, s[tokenStart..i-1]) - - var next = s[i+1] - if next == '{': - # Complex expression: ${foo(bar) in {1..100}} - # Find closing braket, while respecting any nested brackets - inc i - tokenStart = i + 1 - - var - brackets = {'{', '}'} - nestingCount = 1 - - while i < s.len: - inc i, skipUntil(s, brackets, i+1) + 1 - - if not isEscaped(s, i): - if s[i] == '}': - dec nestingCount - if nestingCount == 0: break - else: - inc nestingCount - - yield token(ikExpr, s[tokenStart..(i-1)]) - - tokenStart = i + 1 - + TInterpolatedKind* = enum ## describes for `interpolatedFragments` + ## which part of the interpolated string is + ## yielded; for example in "str$var${expr}" + ikStr, ## ``str`` part of the interpolated string + ikVar, ## ``var`` part of the interpolated string + ikExpr ## ``expr`` part of the interpolated string + +iterator interpolatedFragments*(s: string): tuple[kind: TInterpolatedKind, + value: string] = + ## Tokenizes the string `s` into substrings for interpolation purposes. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## for k, v in interpolatedFragments(" $this is ${an example} "): + ## echo "(", k, ", \"", v, "\")" + ## + ## Results in: + ## + ## .. code-block:: nimrod + ## (ikString, " ") + ## (ikExpr, "this") + ## (ikString, " is ") + ## (ikExpr, "an example") + ## (ikString, " ") + var i = 0 + var kind: TInterpolatedKind + while true: + var j = i + if s[j] == '$' and s[j+1] != '$': + if s[j+1] == '{': + inc j, 2 + var nesting = 0 + while true: + case s[j] + of '{': inc nesting + of '}': + if nesting == 0: + inc j + break + dec nesting + of '\0': + raise newException(EInvalidValue, + "Expected closing '}': " & s[i..s.len]) + else: nil + inc j + inc i, 2 # skip ${ + kind = ikExpr + elif s[j+1] in IdentStartChars: + inc j, 2 + while s[j] in IdentChars: inc(j) + inc i # skip $ + kind = ikVar else: - tokenStart = i + 1 - var identifier = parseIdent(s, i+1) - - if identifier.len > 0: - inc i, identifier.len + raise newException(EInvalidValue, + "Unable to parse a varible name at " & s[i..s.len]) + else: + while j < s.len and (s[j] != '$' or s[j+1] == '$'): inc j + kind = ikStr + if j > i: + # do not copy the trailing } for ikExpr: + yield (kind, substr(s, i, j-1-ord(kind == ikExpr))) + else: + break + i = j - yield token(ikExpr, s[tokenStart..i]) +when isMainModule: + for k, v in interpolatedFragments("$test{} $this is ${an{ example}} "): + echo "(", k, ", \"", v, "\")" - tokenStart = i + 1 - - else: - raise newException(EInvalidValue, "Unable to parse a varible name at " & s[i..s.len]) - - inc i - #end while - - # We've reached the end of the string without finding a new interpolation. - # Return the last fragment at string. - if i != tokenStart: - yield token(ikString, s[tokenStart..i]) {.pop.} diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 9f7e016937..53aa34c83e 100755 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -640,8 +640,7 @@ proc join*(a: openArray[string], sep: string): string {. if len(a) > 0: var L = sep.len * (a.len-1) for i in 0..high(a): inc(L, a[i].len) - result = newString(L) - setLen(result, 0) + result = newStringOfCap(L) add(result, a[0]) for i in 1..high(a): add(result, sep) @@ -655,8 +654,7 @@ proc join*(a: openArray[string]): string {. if len(a) > 0: var L = 0 for i in 0..high(a): inc(L, a[i].len) - result = newString(L) - setLen(result, 0) + result = newStringOfCap(L) for i in 0..high(a): add(result, a[i]) else: result = "" @@ -867,9 +865,9 @@ proc validIdentifier*(s: string): bool {.noSideEffect, proc editDistance*(a, b: string): int {.noSideEffect, rtl, extern: "nsuEditDistance".} = - ## returns the edit distance between `a` and `b`. This uses the Levenshtein - ## distance algorithm with only a linear memory overhead. This implementation - ## is highly optimized! + ## returns the edit distance between `a` and `b`. This uses the + ## `Levenshtein`:idx: distance algorithm with only a linear memory overhead. + ## This implementation is highly optimized! var len1 = a.len var len2 = b.len if len1 > len2: diff --git a/tests/accept/run/tstringinterp.nim b/tests/accept/run/tstringinterp.nim index 55baae7ecd..0d402edc31 100644 --- a/tests/accept/run/tstringinterp.nim +++ b/tests/accept/run/tstringinterp.nim @@ -5,20 +5,17 @@ discard """ import macros, parseutils, strutils -proc concat(strings: openarray[string]) : string = +proc concat(strings: openarray[string]): string = result = newString(0) for s in items(strings): result.add(s) -# This will run though the intee template ProcessInterpolations(e: expr) = - var - s = e[1].strVal - + var s = e[1].strVal for f in interpolatedFragments(s): - if f.kind == ikString: + if f.kind == ikStr: addString(f.value) else: - addExpr(f.value) + addExpr(newCall("$", parseExpr(f.value))) macro formatStyleInterpolation(e: expr): expr = var @@ -41,7 +38,7 @@ macro formatStyleInterpolation(e: expr): expr = result[2] = arrayNode macro concatStyleInterpolation(e: expr): expr = - var args : seq[PNimrodNode] + var args: seq[PNimrodNode] newSeq(args, 0) proc addString(s: string) = args.add(newStrLitNode(s)) @@ -64,8 +61,8 @@ var c = 34 var - s1 = concatStyleInterpolation"Hello ${alice}, ${sum (a, b, c)}}" - s2 = formatStyleInterpolation"Hello ${bob}, ${sum (alice.len, bob.len, 2)}" + s1 = concatStyleInterpolation"Hello ${alice}, ${sum(a, b, c)}" + s2 = formatStyleInterpolation"Hello ${bob}, ${sum(alice.len, bob.len, 2)}" write(stdout, s1 & " | " & s2) diff --git a/todo.txt b/todo.txt index 55c12ba7f5..59f19ab100 100755 --- a/todo.txt +++ b/todo.txt @@ -73,7 +73,6 @@ version 0.9.XX Library ------- -- proper URL-parser - wrappers for poppler; libharu - radix tree for strings; maybe suffix tree - locale support diff --git a/web/news.txt b/web/news.txt index dd30ed7f60..3bd68671fc 100755 --- a/web/news.txt +++ b/web/news.txt @@ -17,6 +17,7 @@ Bugfixes the end of file for text files that do not end with a newline. - Bugfix c2nim, c2pas: the ``--out`` option has never worked properly. - Bugfix: forwarding of generic procs never worked. +- Some more bugfixes for macros and compile-time evaluation. Changes affecting backwards compatibility @@ -80,6 +81,9 @@ Library Additions - Added ``system.running`` for threads. - Added ``xmltree.innerText``. - Added ``os.isAbsolute``. +- Added ``parseutils.interpolatedFragments``. +- Added ``macros.toLisp``, ``macros.toYaml``, ``macros.parseExpr``, + ``macros.parseStmt``, ``macros.getAst``. - Added ``locks`` core module for more flexible locking support. - Added ``irc`` module.