diff --git a/doc/pegdocs.txt b/doc/pegdocs.txt index 6dde731113..05a7fdc584 100755 --- a/doc/pegdocs.txt +++ b/doc/pegdocs.txt @@ -97,6 +97,10 @@ macro meaning ``\n`` any newline combination: ``\10 / \13\10 / \13`` ``\i`` ignore case for matching; use this at the start of the PEG ``\y`` ignore style for matching; use this at the start of the PEG +``\skip`` pat skip pattern *pat* before trying to match other tokens; + this is useful for whitespace skipping, for example: + ``\skip(\s*) {\ident} ':' {\ident}`` matches key value + pairs ignoring whitespace around the ``':'``. ``\ident`` a standard ASCII identifier: ``[a-zA-Z_][a-zA-Z_0-9]*`` ``\letter`` any Unicode letter ``\upper`` any Unicode uppercase letter @@ -198,8 +202,8 @@ As a regular expression ``\[.*\]`` matches the longest possible text between deterministic: ``.*`` consumes the rest of the input, so ``\]`` never matches. As a PEG this needs to be written as: ``\[ ( !\] . )* \]`` (or ``\[ @ \]``). -Note that the regular expression does not behave as intended either: -``*`` should not be greedy, so ``\[.*?\]`` should be used. +Note that the regular expression does not behave as intended either: in the +example ``*`` should not be greedy, so ``\[.*?\]`` should be used instead. PEG construction diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 175713a0c4..f09e8f3b90 100755 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1351,6 +1351,7 @@ type modifier: TModifier captures: int identIsVerbatim: bool + skip: TPeg proc pegError(p: TPegParser, msg: string, line = -1, col = -1) = var e: ref EInvalidPeg @@ -1388,6 +1389,30 @@ proc modifiedBackref(s: int, m: TModifier): TPeg = of modIgnoreCase: result = backRefIgnoreCase(s) of modIgnoreStyle: result = backRefIgnoreStyle(s) +proc builtin(p: var TPegParser): TPeg = + # do not use "y", "skip" or "i" as these would be ambiguous + case p.tok.literal + of "n": result = newLine() + of "d": result = charset({'0'..'9'}) + of "D": result = charset({'\1'..'\xff'} - {'0'..'9'}) + of "s": result = charset({' ', '\9'..'\13'}) + of "S": result = charset({'\1'..'\xff'} - {' ', '\9'..'\13'}) + of "w": result = charset({'a'..'z', 'A'..'Z', '_', '0'..'9'}) + of "W": result = charset({'\1'..'\xff'} - {'a'..'z','A'..'Z','_','0'..'9'}) + of "a": result = charset({'a'..'z', 'A'..'Z'}) + of "A": result = charset({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'}) + of "ident": result = pegs.ident + of "letter": result = UnicodeLetter() + of "upper": result = UnicodeUpper() + of "lower": result = UnicodeLower() + of "title": result = UnicodeTitle() + of "white": result = UnicodeWhitespace() + else: pegError(p, "unknown built-in: " & p.tok.literal) + +proc token(terminal: TPeg, p: TPegParser): TPeg = + if p.skip.kind == pkEmpty: result = terminal + else: result = sequence(p.skip, terminal) + proc primary(p: var TPegParser): TPeg = case p.tok.kind of tkAmp: @@ -1401,31 +1426,31 @@ proc primary(p: var TPegParser): TPeg = return @primary(p) of tkCurlyAt: getTok(p) - return @@primary(p) + return @@primary(p).token(p) else: nil case p.tok.kind of tkIdentifier: if p.identIsVerbatim: var m = p.tok.modifier if m == modNone: m = p.modifier - result = modifiedTerm(p.tok.literal, m) + result = modifiedTerm(p.tok.literal, m).token(p) getTok(p) elif not arrowIsNextTok(p): var nt = getNonTerminal(p, p.tok.literal) incl(nt.flags, ntUsed) - result = nonTerminal(nt) + result = nonTerminal(nt).token(p) getTok(p) else: pegError(p, "expression expected, but found: " & p.tok.literal) of tkStringLit: var m = p.tok.modifier if m == modNone: m = p.modifier - result = modifiedTerm(p.tok.literal, m) + result = modifiedTerm(p.tok.literal, m).token(p) getTok(p) of tkCharSet: if '\0' in p.tok.charset: pegError(p, "binary zero ('\\0') not allowed in character class") - result = charset(p.tok.charset) + result = charset(p.tok.charset).token(p) getTok(p) of tkParLe: getTok(p) @@ -1433,41 +1458,25 @@ proc primary(p: var TPegParser): TPeg = eat(p, tkParRi) of tkCurlyLe: getTok(p) - result = capture(parseExpr(p)) + result = capture(parseExpr(p)).token(p) eat(p, tkCurlyRi) inc(p.captures) of tkAny: - result = any() + result = any().token(p) getTok(p) of tkAnyRune: - result = anyRune() + result = anyRune().token(p) getTok(p) of tkBuiltin: - case p.tok.literal - of "n": result = newLine() - of "d": result = charset({'0'..'9'}) - of "D": result = charset({'\1'..'\xff'} - {'0'..'9'}) - of "s": result = charset({' ', '\9'..'\13'}) - of "S": result = charset({'\1'..'\xff'} - {' ', '\9'..'\13'}) - of "w": result = charset({'a'..'z', 'A'..'Z', '_', '0'..'9'}) - of "W": result = charset({'\1'..'\xff'} - {'a'..'z','A'..'Z','_','0'..'9'}) - of "a": result = charset({'a'..'z', 'A'..'Z'}) - of "A": result = charset({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'}) - of "ident": result = pegs.ident - of "letter": result = UnicodeLetter() - of "upper": result = UnicodeUpper() - of "lower": result = UnicodeLower() - of "title": result = UnicodeTitle() - of "white": result = UnicodeWhitespace() - else: pegError(p, "unknown built-in: " & p.tok.literal) + result = builtin(p).token(p) getTok(p) of tkEscaped: - result = term(p.tok.literal[0]) + result = term(p.tok.literal[0]).token(p) getTok(p) of tkDollar: var m = p.tok.modifier if m == modNone: m = p.modifier - result = modifiedBackRef(p.tok.index, m) + result = modifiedBackRef(p.tok.index, m).token(p) if p.tok.index < 0 or p.tok.index > p.captures: pegError(p, "invalid back reference index: " & $p.tok.index) getTok(p) @@ -1522,7 +1531,7 @@ proc parseRule(p: var TPegParser): PNonTerminal = proc rawParse(p: var TPegParser): TPeg = ## parses a rule or a PEG expression - if p.tok.kind == tkBuiltin: + while p.tok.kind == tkBuiltin: case p.tok.literal of "i": p.modifier = modIgnoreCase @@ -1530,7 +1539,10 @@ proc rawParse(p: var TPegParser): TPeg = of "y": p.modifier = modIgnoreStyle getTok(p) - else: nil + of "skip": + getTok(p) + p.skip = ?primary(p) + else: break if p.tok.kind == tkIdentifier and arrowIsNextTok(p): result = parseRule(p).rule while p.tok.kind != tkEof: @@ -1675,5 +1687,8 @@ when isMainModule: assert match("eine übersicht und auerdem", peg"(\lower \white*)+") assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") assert(not match("456678", peg"(\letter)+")) - - + + assert("var1 = key; var2 = key2".replace( + peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == + "var1<-keykey;var2<-key2key2") + diff --git a/rod/transf.nim b/rod/transf.nim index 88d9ff9842..9eddcbe684 100755 --- a/rod/transf.nim +++ b/rod/transf.nim @@ -188,6 +188,41 @@ proc transformSymAux(c: PTransf, n: PNode): PNode = proc transformSym(c: PTransf, n: PNode): PTransNode = result = PTransNode(transformSymAux(c, n)) +proc transformVarSection(c: PTransf, v: PNode): PTransNode = + result = newTransNode(v) + for i in countup(0, sonsLen(v)-1): + var it = v.sons[i] + if it.kind == nkCommentStmt: + result[i] = PTransNode(it) + elif it.kind == nkIdentDefs: + if (it.sons[0].kind != nkSym): + InternalError(it.info, "transformVarSection") + var newVar = copySym(it.sons[0].sym) + incl(newVar.flags, sfFromGeneric) + # fixes a strange bug for rodgen: + #include(it.sons[0].sym.flags, sfFromGeneric); + newVar.owner = getCurrOwner(c) + IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar)) + var defs = newTransNode(nkIdentDefs, it.info, 3) + defs[0] = newSymNode(newVar).PTransNode + defs[1] = it.sons[1].PTransNode + defs[2] = transform(c, it.sons[2]) + result[i] = defs + else: + if it.kind != nkVarTuple: + InternalError(it.info, "transformVarSection: not nkVarTuple") + var L = sonsLen(it) + var defs = newTransNode(it.kind, it.info, L) + for j in countup(0, L-3): + var newVar = copySym(it.sons[j].sym) + incl(newVar.flags, sfFromGeneric) + newVar.owner = getCurrOwner(c) + IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar)) + defs[j] = newSymNode(newVar).PTransNode + assert(it.sons[L-2] == nil) + defs[L-1] = transform(c, it.sons[L-1]) + result[i] = defs + proc hasContinue(n: PNode): bool = if n == nil: return case n.kind @@ -235,6 +270,21 @@ proc unpackTuple(c: PTransf, n: PNode, father: PTransNode) = add(father, newAsgnStmt(c, c.transCon.forStmt.sons[i], transform(c, newTupleAccess(n, i)))) +proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode = + if n == nil: return + case n.kind + of nkSym: + return transformSym(c, n) + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: + # nothing to be done for leaves: + result = PTransNode(n) + of nkVarSection: + result = transformVarSection(c, n) + else: + result = newTransNode(n) + for i in countup(0, sonsLen(n)-1): + result[i] = introduceNewLocalVars(c, n.sons[i]) + proc transformYield(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkStmtList, n.info, 0) var e = n.sons[0] @@ -255,43 +305,8 @@ proc transformYield(c: PTransf, n: PNode): PTransNode = # common case add(result, c.transCon.forLoopBody) else: - # we need to transform again to introduce new local variables: - add(result, transform(c, c.transCon.forLoopBody.pnode)) - -proc transformVarSection(c: PTransf, v: PNode): PTransNode = - result = newTransNode(v) - for i in countup(0, sonsLen(v)-1): - var it = v.sons[i] - if it.kind == nkCommentStmt: - result[i] = PTransNode(it) - elif it.kind == nkIdentDefs: - if (it.sons[0].kind != nkSym): - InternalError(it.info, "transformVarSection") - var newVar = copySym(it.sons[0].sym) - incl(newVar.flags, sfFromGeneric) - # fixes a strange bug for rodgen: - #include(it.sons[0].sym.flags, sfFromGeneric); - newVar.owner = getCurrOwner(c) - IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar)) - var defs = newTransNode(nkIdentDefs, it.info, 3) - defs[0] = newSymNode(newVar).PTransNode - defs[1] = it.sons[1].PTransNode - defs[2] = transform(c, it.sons[2]) - result[i] = defs - else: - if it.kind != nkVarTuple: - InternalError(it.info, "transformVarSection: not nkVarTuple") - var L = sonsLen(it) - var defs = newTransNode(it.kind, it.info, L) - for j in countup(0, L-3): - var newVar = copySym(it.sons[j].sym) - incl(newVar.flags, sfFromGeneric) - newVar.owner = getCurrOwner(c) - IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar)) - defs[j] = newSymNode(newVar).PTransNode - assert(it.sons[L-2] == nil) - defs[L-1] = transform(c, it.sons[L-1]) - result[i] = defs + # we need to introduce new local variables: + add(result, introduceNewLocalVars(c, c.transCon.forLoopBody.pnode)) proc addVar(father, v: PNode) = var vpart = newNodeI(nkIdentDefs, v.info) diff --git a/tests/tester.nim b/tests/tester.nim index be8f3b0ac8..e9f030115a 100755 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -92,6 +92,9 @@ proc initResults: TResults = result.passed = 0 result.data = "" +proc `$`(x: TResults): string = + result = "Tests passed: " & $x.passed & "/" & $x.total & "
\n" + proc colorBool(b: bool): string = if b: result = "yes" else: result = "no" @@ -116,10 +119,13 @@ proc addResult(r: var TResults, test, given: string, proc listResults(reject, compile, run: TResults) = var s = "" s.add("

Tests to Reject

\n") + s.add($reject) s.add(TableHeader4 & reject.data & TableFooter) s.add("


Tests to Compile

\n") + s.add($compile) s.add(TableHeader3 & compile.data & TableFooter) s.add("


Tests to Run

\n") + s.add($run) s.add(TableHeader4 & run.data & TableFooter) s.add("") var outp: TFile @@ -145,6 +151,8 @@ proc reject(r: var TResults, dir, options: string) = for test in os.walkFiles(dir / "t*.nim"): var t = extractFilename(test) + inc(r.total) + echo t var expected = findSpec(specs, t) var given = callCompiler(test, options) cmpMsgs(r, specs[expected], given, t) @@ -153,6 +161,7 @@ proc compile(r: var TResults, pattern, options: string) = for test in os.walkFiles(pattern): var t = extractFilename(test) inc(r.total) + echo t var given = callCompiler(test, options) r.addResult(t, given.msg, not given.err) if not given.err: inc(r.passed) @@ -162,6 +171,7 @@ proc run(r: var TResults, dir, options: string) = for test in os.walkFiles(dir / "t*.nim"): var t = extractFilename(test) inc(r.total) + echo t var given = callCompiler(test, options) if given.err: r.addResult(t, "", given.msg, not given.err) diff --git a/todo.txt b/todo.txt index 11c40dff74..400aedcdb5 100755 --- a/todo.txt +++ b/todo.txt @@ -1,4 +1,3 @@ -- Tester needs to count failures - we need a way to disable tests diff --git a/web/news.txt b/web/news.txt index 36fdcc79f8..eeff6e3191 100755 --- a/web/news.txt +++ b/web/news.txt @@ -2,6 +2,7 @@ News ==== + 2010-XX-XX Version 0.8.12 released ================================== @@ -29,6 +30,7 @@ Additions - Pegs support a *captured search loop operator* ``{@}``. - Pegs support new built-ins: ``\letter``, ``\upper``, ``\lower``, ``\title``, ``\white``. +- Pegs support the new built-in ``\skip`` operation. 2010-10-20 Version 0.8.10 released