diff --git a/doc/pegdocs.txt b/doc/pegdocs.txt
index 6dde731113..05a7fdc584 100755
--- a/doc/pegdocs.txt
+++ b/doc/pegdocs.txt
@@ -97,6 +97,10 @@ macro meaning
``\n`` any newline combination: ``\10 / \13\10 / \13``
``\i`` ignore case for matching; use this at the start of the PEG
``\y`` ignore style for matching; use this at the start of the PEG
+``\skip`` pat skip pattern *pat* before trying to match other tokens;
+ this is useful for whitespace skipping, for example:
+ ``\skip(\s*) {\ident} ':' {\ident}`` matches key value
+ pairs ignoring whitespace around the ``':'``.
``\ident`` a standard ASCII identifier: ``[a-zA-Z_][a-zA-Z_0-9]*``
``\letter`` any Unicode letter
``\upper`` any Unicode uppercase letter
@@ -198,8 +202,8 @@ As a regular expression ``\[.*\]`` matches the longest possible text between
deterministic: ``.*`` consumes the rest of the input, so ``\]`` never matches.
As a PEG this needs to be written as: ``\[ ( !\] . )* \]`` (or ``\[ @ \]``).
-Note that the regular expression does not behave as intended either:
-``*`` should not be greedy, so ``\[.*?\]`` should be used.
+Note that the regular expression does not behave as intended either: in the
+example ``*`` should not be greedy, so ``\[.*?\]`` should be used instead.
PEG construction
diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim
index 175713a0c4..f09e8f3b90 100755
--- a/lib/pure/pegs.nim
+++ b/lib/pure/pegs.nim
@@ -1351,6 +1351,7 @@ type
modifier: TModifier
captures: int
identIsVerbatim: bool
+ skip: TPeg
proc pegError(p: TPegParser, msg: string, line = -1, col = -1) =
var e: ref EInvalidPeg
@@ -1388,6 +1389,30 @@ proc modifiedBackref(s: int, m: TModifier): TPeg =
of modIgnoreCase: result = backRefIgnoreCase(s)
of modIgnoreStyle: result = backRefIgnoreStyle(s)
+proc builtin(p: var TPegParser): TPeg =
+ # do not use "y", "skip" or "i" as these would be ambiguous
+ case p.tok.literal
+ of "n": result = newLine()
+ of "d": result = charset({'0'..'9'})
+ of "D": result = charset({'\1'..'\xff'} - {'0'..'9'})
+ of "s": result = charset({' ', '\9'..'\13'})
+ of "S": result = charset({'\1'..'\xff'} - {' ', '\9'..'\13'})
+ of "w": result = charset({'a'..'z', 'A'..'Z', '_', '0'..'9'})
+ of "W": result = charset({'\1'..'\xff'} - {'a'..'z','A'..'Z','_','0'..'9'})
+ of "a": result = charset({'a'..'z', 'A'..'Z'})
+ of "A": result = charset({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'})
+ of "ident": result = pegs.ident
+ of "letter": result = UnicodeLetter()
+ of "upper": result = UnicodeUpper()
+ of "lower": result = UnicodeLower()
+ of "title": result = UnicodeTitle()
+ of "white": result = UnicodeWhitespace()
+ else: pegError(p, "unknown built-in: " & p.tok.literal)
+
+proc token(terminal: TPeg, p: TPegParser): TPeg =
+ if p.skip.kind == pkEmpty: result = terminal
+ else: result = sequence(p.skip, terminal)
+
proc primary(p: var TPegParser): TPeg =
case p.tok.kind
of tkAmp:
@@ -1401,31 +1426,31 @@ proc primary(p: var TPegParser): TPeg =
return @primary(p)
of tkCurlyAt:
getTok(p)
- return @@primary(p)
+ return @@primary(p).token(p)
else: nil
case p.tok.kind
of tkIdentifier:
if p.identIsVerbatim:
var m = p.tok.modifier
if m == modNone: m = p.modifier
- result = modifiedTerm(p.tok.literal, m)
+ result = modifiedTerm(p.tok.literal, m).token(p)
getTok(p)
elif not arrowIsNextTok(p):
var nt = getNonTerminal(p, p.tok.literal)
incl(nt.flags, ntUsed)
- result = nonTerminal(nt)
+ result = nonTerminal(nt).token(p)
getTok(p)
else:
pegError(p, "expression expected, but found: " & p.tok.literal)
of tkStringLit:
var m = p.tok.modifier
if m == modNone: m = p.modifier
- result = modifiedTerm(p.tok.literal, m)
+ result = modifiedTerm(p.tok.literal, m).token(p)
getTok(p)
of tkCharSet:
if '\0' in p.tok.charset:
pegError(p, "binary zero ('\\0') not allowed in character class")
- result = charset(p.tok.charset)
+ result = charset(p.tok.charset).token(p)
getTok(p)
of tkParLe:
getTok(p)
@@ -1433,41 +1458,25 @@ proc primary(p: var TPegParser): TPeg =
eat(p, tkParRi)
of tkCurlyLe:
getTok(p)
- result = capture(parseExpr(p))
+ result = capture(parseExpr(p)).token(p)
eat(p, tkCurlyRi)
inc(p.captures)
of tkAny:
- result = any()
+ result = any().token(p)
getTok(p)
of tkAnyRune:
- result = anyRune()
+ result = anyRune().token(p)
getTok(p)
of tkBuiltin:
- case p.tok.literal
- of "n": result = newLine()
- of "d": result = charset({'0'..'9'})
- of "D": result = charset({'\1'..'\xff'} - {'0'..'9'})
- of "s": result = charset({' ', '\9'..'\13'})
- of "S": result = charset({'\1'..'\xff'} - {' ', '\9'..'\13'})
- of "w": result = charset({'a'..'z', 'A'..'Z', '_', '0'..'9'})
- of "W": result = charset({'\1'..'\xff'} - {'a'..'z','A'..'Z','_','0'..'9'})
- of "a": result = charset({'a'..'z', 'A'..'Z'})
- of "A": result = charset({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'})
- of "ident": result = pegs.ident
- of "letter": result = UnicodeLetter()
- of "upper": result = UnicodeUpper()
- of "lower": result = UnicodeLower()
- of "title": result = UnicodeTitle()
- of "white": result = UnicodeWhitespace()
- else: pegError(p, "unknown built-in: " & p.tok.literal)
+ result = builtin(p).token(p)
getTok(p)
of tkEscaped:
- result = term(p.tok.literal[0])
+ result = term(p.tok.literal[0]).token(p)
getTok(p)
of tkDollar:
var m = p.tok.modifier
if m == modNone: m = p.modifier
- result = modifiedBackRef(p.tok.index, m)
+ result = modifiedBackRef(p.tok.index, m).token(p)
if p.tok.index < 0 or p.tok.index > p.captures:
pegError(p, "invalid back reference index: " & $p.tok.index)
getTok(p)
@@ -1522,7 +1531,7 @@ proc parseRule(p: var TPegParser): PNonTerminal =
proc rawParse(p: var TPegParser): TPeg =
## parses a rule or a PEG expression
- if p.tok.kind == tkBuiltin:
+ while p.tok.kind == tkBuiltin:
case p.tok.literal
of "i":
p.modifier = modIgnoreCase
@@ -1530,7 +1539,10 @@ proc rawParse(p: var TPegParser): TPeg =
of "y":
p.modifier = modIgnoreStyle
getTok(p)
- else: nil
+ of "skip":
+ getTok(p)
+ p.skip = ?primary(p)
+ else: break
if p.tok.kind == tkIdentifier and arrowIsNextTok(p):
result = parseRule(p).rule
while p.tok.kind != tkEof:
@@ -1675,5 +1687,8 @@ when isMainModule:
assert match("eine übersicht und auerdem", peg"(\lower \white*)+")
assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+")
assert(not match("456678", peg"(\letter)+"))
-
-
+
+ assert("var1 = key; var2 = key2".replace(
+ peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") ==
+ "var1<-keykey;var2<-key2key2")
+
diff --git a/rod/transf.nim b/rod/transf.nim
index 88d9ff9842..9eddcbe684 100755
--- a/rod/transf.nim
+++ b/rod/transf.nim
@@ -188,6 +188,41 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
proc transformSym(c: PTransf, n: PNode): PTransNode =
result = PTransNode(transformSymAux(c, n))
+proc transformVarSection(c: PTransf, v: PNode): PTransNode =
+ result = newTransNode(v)
+ for i in countup(0, sonsLen(v)-1):
+ var it = v.sons[i]
+ if it.kind == nkCommentStmt:
+ result[i] = PTransNode(it)
+ elif it.kind == nkIdentDefs:
+ if (it.sons[0].kind != nkSym):
+ InternalError(it.info, "transformVarSection")
+ var newVar = copySym(it.sons[0].sym)
+ incl(newVar.flags, sfFromGeneric)
+ # fixes a strange bug for rodgen:
+ #include(it.sons[0].sym.flags, sfFromGeneric);
+ newVar.owner = getCurrOwner(c)
+ IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
+ var defs = newTransNode(nkIdentDefs, it.info, 3)
+ defs[0] = newSymNode(newVar).PTransNode
+ defs[1] = it.sons[1].PTransNode
+ defs[2] = transform(c, it.sons[2])
+ result[i] = defs
+ else:
+ if it.kind != nkVarTuple:
+ InternalError(it.info, "transformVarSection: not nkVarTuple")
+ var L = sonsLen(it)
+ var defs = newTransNode(it.kind, it.info, L)
+ for j in countup(0, L-3):
+ var newVar = copySym(it.sons[j].sym)
+ incl(newVar.flags, sfFromGeneric)
+ newVar.owner = getCurrOwner(c)
+ IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar))
+ defs[j] = newSymNode(newVar).PTransNode
+ assert(it.sons[L-2] == nil)
+ defs[L-1] = transform(c, it.sons[L-1])
+ result[i] = defs
+
proc hasContinue(n: PNode): bool =
if n == nil: return
case n.kind
@@ -235,6 +270,21 @@ proc unpackTuple(c: PTransf, n: PNode, father: PTransNode) =
add(father, newAsgnStmt(c, c.transCon.forStmt.sons[i],
transform(c, newTupleAccess(n, i))))
+proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode =
+ if n == nil: return
+ case n.kind
+ of nkSym:
+ return transformSym(c, n)
+ of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit:
+ # nothing to be done for leaves:
+ result = PTransNode(n)
+ of nkVarSection:
+ result = transformVarSection(c, n)
+ else:
+ result = newTransNode(n)
+ for i in countup(0, sonsLen(n)-1):
+ result[i] = introduceNewLocalVars(c, n.sons[i])
+
proc transformYield(c: PTransf, n: PNode): PTransNode =
result = newTransNode(nkStmtList, n.info, 0)
var e = n.sons[0]
@@ -255,43 +305,8 @@ proc transformYield(c: PTransf, n: PNode): PTransNode =
# common case
add(result, c.transCon.forLoopBody)
else:
- # we need to transform again to introduce new local variables:
- add(result, transform(c, c.transCon.forLoopBody.pnode))
-
-proc transformVarSection(c: PTransf, v: PNode): PTransNode =
- result = newTransNode(v)
- for i in countup(0, sonsLen(v)-1):
- var it = v.sons[i]
- if it.kind == nkCommentStmt:
- result[i] = PTransNode(it)
- elif it.kind == nkIdentDefs:
- if (it.sons[0].kind != nkSym):
- InternalError(it.info, "transformVarSection")
- var newVar = copySym(it.sons[0].sym)
- incl(newVar.flags, sfFromGeneric)
- # fixes a strange bug for rodgen:
- #include(it.sons[0].sym.flags, sfFromGeneric);
- newVar.owner = getCurrOwner(c)
- IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
- var defs = newTransNode(nkIdentDefs, it.info, 3)
- defs[0] = newSymNode(newVar).PTransNode
- defs[1] = it.sons[1].PTransNode
- defs[2] = transform(c, it.sons[2])
- result[i] = defs
- else:
- if it.kind != nkVarTuple:
- InternalError(it.info, "transformVarSection: not nkVarTuple")
- var L = sonsLen(it)
- var defs = newTransNode(it.kind, it.info, L)
- for j in countup(0, L-3):
- var newVar = copySym(it.sons[j].sym)
- incl(newVar.flags, sfFromGeneric)
- newVar.owner = getCurrOwner(c)
- IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar))
- defs[j] = newSymNode(newVar).PTransNode
- assert(it.sons[L-2] == nil)
- defs[L-1] = transform(c, it.sons[L-1])
- result[i] = defs
+ # we need to introduce new local variables:
+ add(result, introduceNewLocalVars(c, c.transCon.forLoopBody.pnode))
proc addVar(father, v: PNode) =
var vpart = newNodeI(nkIdentDefs, v.info)
diff --git a/tests/tester.nim b/tests/tester.nim
index be8f3b0ac8..e9f030115a 100755
--- a/tests/tester.nim
+++ b/tests/tester.nim
@@ -92,6 +92,9 @@ proc initResults: TResults =
result.passed = 0
result.data = ""
+proc `$`(x: TResults): string =
+ result = "Tests passed: " & $x.passed & "/" & $x.total & "
\n"
+
proc colorBool(b: bool): string =
if b: result = "yes"
else: result = "no"
@@ -116,10 +119,13 @@ proc addResult(r: var TResults, test, given: string,
proc listResults(reject, compile, run: TResults) =
var s = ""
s.add("