|
|
|
|
@@ -24,7 +24,7 @@ when useUnicode:
|
|
|
|
|
const
|
|
|
|
|
InlineThreshold = 5 ## number of leaves; -1 to disable inlining
|
|
|
|
|
MaxSubpatterns* = 10 ## defines the maximum number of subpatterns that
|
|
|
|
|
## can be captured. More subpatterns cannot be captured!
|
|
|
|
|
## can be captured. More subpatterns cannot be captured!
|
|
|
|
|
|
|
|
|
|
type
|
|
|
|
|
TPegKind = enum
|
|
|
|
|
@@ -80,12 +80,12 @@ type
|
|
|
|
|
of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns]
|
|
|
|
|
else: sons: seq[TNode]
|
|
|
|
|
PNonTerminal* = ref TNonTerminal
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TPeg* = TNode ## type that represents a PEG
|
|
|
|
|
|
|
|
|
|
proc term*(t: string): TPeg {.rtl, extern: "npegs$1Str".} =
|
|
|
|
|
## constructs a PEG from a terminal string
|
|
|
|
|
if t.len != 1:
|
|
|
|
|
if t.len != 1:
|
|
|
|
|
result.kind = pkTerminal
|
|
|
|
|
result.term = t
|
|
|
|
|
else:
|
|
|
|
|
@@ -109,7 +109,7 @@ proc term*(t: char): TPeg {.rtl, extern: "npegs$1Char".} =
|
|
|
|
|
assert t != '\0'
|
|
|
|
|
result.kind = pkChar
|
|
|
|
|
result.ch = t
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc charSet*(s: set[char]): TPeg {.rtl, extern: "npegs$1".} =
|
|
|
|
|
## constructs a PEG from a character set `s`
|
|
|
|
|
assert '\0' notin s
|
|
|
|
|
@@ -120,31 +120,31 @@ proc charSet*(s: set[char]): TPeg {.rtl, extern: "npegs$1".} =
|
|
|
|
|
proc len(a: TPeg): int {.inline.} = return a.sons.len
|
|
|
|
|
proc add(d: var TPeg, s: TPeg) {.inline.} = add(d.sons, s)
|
|
|
|
|
|
|
|
|
|
proc copyPeg(a: TPeg): TPeg =
|
|
|
|
|
proc copyPeg(a: TPeg): TPeg =
|
|
|
|
|
result.kind = a.kind
|
|
|
|
|
case a.kind
|
|
|
|
|
of pkEmpty..pkWhitespace: discard
|
|
|
|
|
of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle:
|
|
|
|
|
of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle:
|
|
|
|
|
result.term = a.term
|
|
|
|
|
of pkChar, pkGreedyRepChar:
|
|
|
|
|
of pkChar, pkGreedyRepChar:
|
|
|
|
|
result.ch = a.ch
|
|
|
|
|
of pkCharChoice, pkGreedyRepSet:
|
|
|
|
|
of pkCharChoice, pkGreedyRepSet:
|
|
|
|
|
new(result.charChoice)
|
|
|
|
|
result.charChoice[] = a.charChoice[]
|
|
|
|
|
of pkNonTerminal: result.nt = a.nt
|
|
|
|
|
of pkBackRef..pkBackRefIgnoreStyle:
|
|
|
|
|
of pkBackRef..pkBackRefIgnoreStyle:
|
|
|
|
|
result.index = a.index
|
|
|
|
|
else:
|
|
|
|
|
else:
|
|
|
|
|
result.sons = a.sons
|
|
|
|
|
|
|
|
|
|
proc addChoice(dest: var TPeg, elem: TPeg) =
|
|
|
|
|
var L = dest.len-1
|
|
|
|
|
if L >= 0 and dest.sons[L].kind == pkCharChoice:
|
|
|
|
|
if L >= 0 and dest.sons[L].kind == pkCharChoice:
|
|
|
|
|
# caution! Do not introduce false aliasing here!
|
|
|
|
|
case elem.kind
|
|
|
|
|
of pkCharChoice:
|
|
|
|
|
dest.sons[L] = charSet(dest.sons[L].charChoice[] + elem.charChoice[])
|
|
|
|
|
of pkChar:
|
|
|
|
|
of pkChar:
|
|
|
|
|
dest.sons[L] = charSet(dest.sons[L].charChoice[] + {elem.ch})
|
|
|
|
|
else: add(dest, elem)
|
|
|
|
|
else: add(dest, elem)
|
|
|
|
|
@@ -168,12 +168,12 @@ proc `/`*(a: varargs[TPeg]): TPeg {.
|
|
|
|
|
|
|
|
|
|
proc addSequence(dest: var TPeg, elem: TPeg) =
|
|
|
|
|
var L = dest.len-1
|
|
|
|
|
if L >= 0 and dest.sons[L].kind == pkTerminal:
|
|
|
|
|
if L >= 0 and dest.sons[L].kind == pkTerminal:
|
|
|
|
|
# caution! Do not introduce false aliasing here!
|
|
|
|
|
case elem.kind
|
|
|
|
|
of pkTerminal:
|
|
|
|
|
of pkTerminal:
|
|
|
|
|
dest.sons[L] = term(dest.sons[L].term & elem.term)
|
|
|
|
|
of pkChar:
|
|
|
|
|
of pkChar:
|
|
|
|
|
dest.sons[L] = term(dest.sons[L].term & elem.ch)
|
|
|
|
|
else: add(dest, elem)
|
|
|
|
|
else: add(dest, elem)
|
|
|
|
|
@@ -182,7 +182,7 @@ proc sequence*(a: varargs[TPeg]): TPeg {.
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
## constructs a sequence with all the PEGs from `a`
|
|
|
|
|
multipleOp(pkSequence, addSequence)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc `?`*(a: TPeg): TPeg {.rtl, extern: "npegsOptional".} =
|
|
|
|
|
## constructs an optional for the PEG `a`
|
|
|
|
|
if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar,
|
|
|
|
|
@@ -217,12 +217,12 @@ proc `!*`*(a: TPeg): TPeg {.rtl, extern: "npegsSearch".} =
|
|
|
|
|
result.kind = pkSearch
|
|
|
|
|
result.sons = @[a]
|
|
|
|
|
|
|
|
|
|
proc `!*\`*(a: TPeg): TPeg {.rtl,
|
|
|
|
|
proc `!*\`*(a: TPeg): TPeg {.rtl,
|
|
|
|
|
extern: "npgegsCapturedSearch".} =
|
|
|
|
|
## constructs a "captured search" for the PEG `a`
|
|
|
|
|
result.kind = pkCapturedSearch
|
|
|
|
|
result.sons = @[a]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
when false:
|
|
|
|
|
proc contains(a: TPeg, k: TPegKind): bool =
|
|
|
|
|
if a.kind == k: return true
|
|
|
|
|
@@ -238,7 +238,7 @@ when false:
|
|
|
|
|
proc `+`*(a: TPeg): TPeg {.rtl, extern: "npegsGreedyPosRep".} =
|
|
|
|
|
## constructs a "greedy positive repetition" with the PEG `a`
|
|
|
|
|
return sequence(a, *a)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc `&`*(a: TPeg): TPeg {.rtl, extern: "npegsAndPredicate".} =
|
|
|
|
|
## constructs an "and predicate" with the PEG `a`
|
|
|
|
|
result.kind = pkAndPredicate
|
|
|
|
|
@@ -261,33 +261,33 @@ proc newLine*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG `newline`:idx: (``\n``)
|
|
|
|
|
result.kind = pkNewline
|
|
|
|
|
|
|
|
|
|
proc UnicodeLetter*: TPeg {.inline.} =
|
|
|
|
|
proc UnicodeLetter*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``\letter`` which matches any Unicode letter.
|
|
|
|
|
result.kind = pkLetter
|
|
|
|
|
|
|
|
|
|
proc UnicodeLower*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``\lower`` which matches any Unicode lowercase letter.
|
|
|
|
|
result.kind = pkLower
|
|
|
|
|
|
|
|
|
|
proc UnicodeUpper*: TPeg {.inline.} =
|
|
|
|
|
proc UnicodeLower*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``\lower`` which matches any Unicode lowercase letter.
|
|
|
|
|
result.kind = pkLower
|
|
|
|
|
|
|
|
|
|
proc UnicodeUpper*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``\upper`` which matches any Unicode lowercase letter.
|
|
|
|
|
result.kind = pkUpper
|
|
|
|
|
|
|
|
|
|
proc UnicodeTitle*: TPeg {.inline.} =
|
|
|
|
|
result.kind = pkUpper
|
|
|
|
|
|
|
|
|
|
proc UnicodeTitle*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``\title`` which matches any Unicode title letter.
|
|
|
|
|
result.kind = pkTitle
|
|
|
|
|
|
|
|
|
|
proc UnicodeWhitespace*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``\white`` which matches any Unicode
|
|
|
|
|
proc UnicodeWhitespace*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``\white`` which matches any Unicode
|
|
|
|
|
## whitespace character.
|
|
|
|
|
result.kind = pkWhitespace
|
|
|
|
|
|
|
|
|
|
proc startAnchor*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``^`` which matches the start of the input.
|
|
|
|
|
proc startAnchor*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``^`` which matches the start of the input.
|
|
|
|
|
result.kind = pkStartAnchor
|
|
|
|
|
|
|
|
|
|
proc endAnchor*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``$`` which matches the end of the input.
|
|
|
|
|
proc endAnchor*: TPeg {.inline.} =
|
|
|
|
|
## constructs the PEG ``$`` which matches the end of the input.
|
|
|
|
|
result = !any()
|
|
|
|
|
|
|
|
|
|
proc capture*(a: TPeg): TPeg {.rtl, extern: "npegsCapture".} =
|
|
|
|
|
@@ -296,21 +296,21 @@ proc capture*(a: TPeg): TPeg {.rtl, extern: "npegsCapture".} =
|
|
|
|
|
result.sons = @[a]
|
|
|
|
|
|
|
|
|
|
proc backref*(index: range[1..MaxSubPatterns]): TPeg {.
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
## constructs a back reference of the given `index`. `index` starts counting
|
|
|
|
|
## from 1.
|
|
|
|
|
result.kind = pkBackRef
|
|
|
|
|
result.index = index-1
|
|
|
|
|
|
|
|
|
|
proc backrefIgnoreCase*(index: range[1..MaxSubPatterns]): TPeg {.
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
## constructs a back reference of the given `index`. `index` starts counting
|
|
|
|
|
## from 1. Ignores case for matching.
|
|
|
|
|
result.kind = pkBackRefIgnoreCase
|
|
|
|
|
result.index = index-1
|
|
|
|
|
|
|
|
|
|
proc backrefIgnoreStyle*(index: range[1..MaxSubPatterns]): TPeg {.
|
|
|
|
|
rtl, extern: "npegs$1".}=
|
|
|
|
|
rtl, extern: "npegs$1".}=
|
|
|
|
|
## constructs a back reference of the given `index`. `index` starts counting
|
|
|
|
|
## from 1. Ignores style for matching.
|
|
|
|
|
result.kind = pkBackRefIgnoreStyle
|
|
|
|
|
@@ -320,7 +320,7 @@ proc spaceCost(n: TPeg): int =
|
|
|
|
|
case n.kind
|
|
|
|
|
of pkEmpty: discard
|
|
|
|
|
of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar,
|
|
|
|
|
pkGreedyRepChar, pkCharChoice, pkGreedyRepSet,
|
|
|
|
|
pkGreedyRepChar, pkCharChoice, pkGreedyRepSet,
|
|
|
|
|
pkAny..pkWhitespace, pkGreedyAny:
|
|
|
|
|
result = 1
|
|
|
|
|
of pkNonTerminal:
|
|
|
|
|
@@ -332,7 +332,7 @@ proc spaceCost(n: TPeg): int =
|
|
|
|
|
if result >= InlineThreshold: break
|
|
|
|
|
|
|
|
|
|
proc nonterminal*(n: PNonTerminal): TPeg {.
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
## constructs a PEG that consists of the nonterminal symbol
|
|
|
|
|
assert n != nil
|
|
|
|
|
if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold:
|
|
|
|
|
@@ -353,7 +353,7 @@ proc newNonTerminal*(name: string, line, column: int): PNonTerminal {.
|
|
|
|
|
template letters*: expr =
|
|
|
|
|
## expands to ``charset({'A'..'Z', 'a'..'z'})``
|
|
|
|
|
charset({'A'..'Z', 'a'..'z'})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template digits*: expr =
|
|
|
|
|
## expands to ``charset({'0'..'9'})``
|
|
|
|
|
charset({'0'..'9'})
|
|
|
|
|
@@ -361,11 +361,11 @@ template digits*: expr =
|
|
|
|
|
template whitespace*: expr =
|
|
|
|
|
## expands to ``charset({' ', '\9'..'\13'})``
|
|
|
|
|
charset({' ', '\9'..'\13'})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template identChars*: expr =
|
|
|
|
|
## expands to ``charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})``
|
|
|
|
|
charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template identStartChars*: expr =
|
|
|
|
|
## expands to ``charset({'A'..'Z', 'a'..'z', '_'})``
|
|
|
|
|
charset({'a'..'z', 'A'..'Z', '_'})
|
|
|
|
|
@@ -374,14 +374,14 @@ template ident*: expr =
|
|
|
|
|
## same as ``[a-zA-Z_][a-zA-z_0-9]*``; standard identifier
|
|
|
|
|
sequence(charset({'a'..'z', 'A'..'Z', '_'}),
|
|
|
|
|
*charset({'a'..'z', 'A'..'Z', '0'..'9', '_'}))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template natural*: expr =
|
|
|
|
|
## same as ``\d+``
|
|
|
|
|
+digits
|
|
|
|
|
|
|
|
|
|
# ------------------------- debugging -----------------------------------------
|
|
|
|
|
|
|
|
|
|
proc esc(c: char, reserved = {'\0'..'\255'}): string =
|
|
|
|
|
proc esc(c: char, reserved = {'\0'..'\255'}): string =
|
|
|
|
|
case c
|
|
|
|
|
of '\b': result = "\\b"
|
|
|
|
|
of '\t': result = "\\t"
|
|
|
|
|
@@ -396,38 +396,38 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string =
|
|
|
|
|
elif c < ' ' or c >= '\128': result = '\\' & $ord(c)
|
|
|
|
|
elif c in reserved: result = '\\' & c
|
|
|
|
|
else: result = $c
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'"
|
|
|
|
|
|
|
|
|
|
proc singleQuoteEsc(str: string): string =
|
|
|
|
|
proc singleQuoteEsc(str: string): string =
|
|
|
|
|
result = "'"
|
|
|
|
|
for c in items(str): add result, esc(c, {'\''})
|
|
|
|
|
add result, '\''
|
|
|
|
|
|
|
|
|
|
proc charSetEscAux(cc: set[char]): string =
|
|
|
|
|
|
|
|
|
|
proc charSetEscAux(cc: set[char]): string =
|
|
|
|
|
const reserved = {'^', '-', ']'}
|
|
|
|
|
result = ""
|
|
|
|
|
var c1 = 0
|
|
|
|
|
while c1 <= 0xff:
|
|
|
|
|
if chr(c1) in cc:
|
|
|
|
|
while c1 <= 0xff:
|
|
|
|
|
if chr(c1) in cc:
|
|
|
|
|
var c2 = c1
|
|
|
|
|
while c2 < 0xff and chr(succ(c2)) in cc: inc(c2)
|
|
|
|
|
if c1 == c2:
|
|
|
|
|
if c1 == c2:
|
|
|
|
|
add result, esc(chr(c1), reserved)
|
|
|
|
|
elif c2 == succ(c1):
|
|
|
|
|
elif c2 == succ(c1):
|
|
|
|
|
add result, esc(chr(c1), reserved) & esc(chr(c2), reserved)
|
|
|
|
|
else:
|
|
|
|
|
else:
|
|
|
|
|
add result, esc(chr(c1), reserved) & '-' & esc(chr(c2), reserved)
|
|
|
|
|
c1 = c2
|
|
|
|
|
inc(c1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc charSetEsc(cc: set[char]): string =
|
|
|
|
|
if card(cc) >= 128+64:
|
|
|
|
|
if card(cc) >= 128+64:
|
|
|
|
|
result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']'
|
|
|
|
|
else:
|
|
|
|
|
else:
|
|
|
|
|
result = '[' & charSetEscAux(cc) & ']'
|
|
|
|
|
|
|
|
|
|
proc toStrAux(r: TPeg, res: var string) =
|
|
|
|
|
|
|
|
|
|
proc toStrAux(r: TPeg, res: var string) =
|
|
|
|
|
case r.kind
|
|
|
|
|
of pkEmpty: add(res, "()")
|
|
|
|
|
of pkAny: add(res, '.')
|
|
|
|
|
@@ -491,25 +491,25 @@ proc toStrAux(r: TPeg, res: var string) =
|
|
|
|
|
toStrAux(r.sons[0], res)
|
|
|
|
|
of pkCapture:
|
|
|
|
|
add(res, '{')
|
|
|
|
|
toStrAux(r.sons[0], res)
|
|
|
|
|
toStrAux(r.sons[0], res)
|
|
|
|
|
add(res, '}')
|
|
|
|
|
of pkBackRef:
|
|
|
|
|
of pkBackRef:
|
|
|
|
|
add(res, '$')
|
|
|
|
|
add(res, $r.index)
|
|
|
|
|
of pkBackRefIgnoreCase:
|
|
|
|
|
of pkBackRefIgnoreCase:
|
|
|
|
|
add(res, "i$")
|
|
|
|
|
add(res, $r.index)
|
|
|
|
|
of pkBackRefIgnoreStyle:
|
|
|
|
|
of pkBackRefIgnoreStyle:
|
|
|
|
|
add(res, "y$")
|
|
|
|
|
add(res, $r.index)
|
|
|
|
|
of pkRule:
|
|
|
|
|
toStrAux(r.sons[0], res)
|
|
|
|
|
toStrAux(r.sons[0], res)
|
|
|
|
|
add(res, " <- ")
|
|
|
|
|
toStrAux(r.sons[1], res)
|
|
|
|
|
of pkList:
|
|
|
|
|
for i in 0 .. high(r.sons):
|
|
|
|
|
toStrAux(r.sons[i], res)
|
|
|
|
|
add(res, "\n")
|
|
|
|
|
add(res, "\n")
|
|
|
|
|
of pkStartAnchor:
|
|
|
|
|
add(res, '^')
|
|
|
|
|
|
|
|
|
|
@@ -526,8 +526,8 @@ type
|
|
|
|
|
ml: int
|
|
|
|
|
origStart: int
|
|
|
|
|
|
|
|
|
|
proc bounds*(c: TCaptures,
|
|
|
|
|
i: range[0..MaxSubpatterns-1]): tuple[first, last: int] =
|
|
|
|
|
proc bounds*(c: TCaptures,
|
|
|
|
|
i: range[0..MaxSubpatterns-1]): tuple[first, last: int] =
|
|
|
|
|
## returns the bounds ``[first..last]`` of the `i`'th capture.
|
|
|
|
|
result = c.matches[i]
|
|
|
|
|
|
|
|
|
|
@@ -547,7 +547,7 @@ when not useUnicode:
|
|
|
|
|
|
|
|
|
|
proc rawMatch*(s: string, p: TPeg, start: int, c: var TCaptures): int {.
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
## low-level matching proc that implements the PEG interpreter. Use this
|
|
|
|
|
## low-level matching proc that implements the PEG interpreter. Use this
|
|
|
|
|
## for maximum efficiency (every other PEG operation ends up calling this
|
|
|
|
|
## proc).
|
|
|
|
|
## Returns -1 if it does not match, else the length of the match
|
|
|
|
|
@@ -561,7 +561,7 @@ proc rawMatch*(s: string, p: TPeg, start: int, c: var TCaptures): int {.
|
|
|
|
|
result = runeLenAt(s, start)
|
|
|
|
|
else:
|
|
|
|
|
result = -1
|
|
|
|
|
of pkLetter:
|
|
|
|
|
of pkLetter:
|
|
|
|
|
if s[start] != '\0':
|
|
|
|
|
var a: Rune
|
|
|
|
|
result = start
|
|
|
|
|
@@ -570,7 +570,7 @@ proc rawMatch*(s: string, p: TPeg, start: int, c: var TCaptures): int {.
|
|
|
|
|
else: result = -1
|
|
|
|
|
else:
|
|
|
|
|
result = -1
|
|
|
|
|
of pkLower:
|
|
|
|
|
of pkLower:
|
|
|
|
|
if s[start] != '\0':
|
|
|
|
|
var a: Rune
|
|
|
|
|
result = start
|
|
|
|
|
@@ -579,7 +579,7 @@ proc rawMatch*(s: string, p: TPeg, start: int, c: var TCaptures): int {.
|
|
|
|
|
else: result = -1
|
|
|
|
|
else:
|
|
|
|
|
result = -1
|
|
|
|
|
of pkUpper:
|
|
|
|
|
of pkUpper:
|
|
|
|
|
if s[start] != '\0':
|
|
|
|
|
var a: Rune
|
|
|
|
|
result = start
|
|
|
|
|
@@ -588,16 +588,16 @@ proc rawMatch*(s: string, p: TPeg, start: int, c: var TCaptures): int {.
|
|
|
|
|
else: result = -1
|
|
|
|
|
else:
|
|
|
|
|
result = -1
|
|
|
|
|
of pkTitle:
|
|
|
|
|
of pkTitle:
|
|
|
|
|
if s[start] != '\0':
|
|
|
|
|
var a: Rune
|
|
|
|
|
result = start
|
|
|
|
|
fastRuneAt(s, result, a)
|
|
|
|
|
if isTitle(a): dec(result, start)
|
|
|
|
|
if isTitle(a): dec(result, start)
|
|
|
|
|
else: result = -1
|
|
|
|
|
else:
|
|
|
|
|
result = -1
|
|
|
|
|
of pkWhitespace:
|
|
|
|
|
of pkWhitespace:
|
|
|
|
|
if s[start] != '\0':
|
|
|
|
|
var a: Rune
|
|
|
|
|
result = start
|
|
|
|
|
@@ -661,7 +661,7 @@ proc rawMatch*(s: string, p: TPeg, start: int, c: var TCaptures): int {.
|
|
|
|
|
when false: echo "leave: ", p.nt.name
|
|
|
|
|
if result < 0: c.ml = oldMl
|
|
|
|
|
of pkSequence:
|
|
|
|
|
var oldMl = c.ml
|
|
|
|
|
var oldMl = c.ml
|
|
|
|
|
result = 0
|
|
|
|
|
assert(not isNil(p.sons))
|
|
|
|
|
for i in 0..high(p.sons):
|
|
|
|
|
@@ -744,11 +744,11 @@ proc rawMatch*(s: string, p: TPeg, start: int, c: var TCaptures): int {.
|
|
|
|
|
#else: silently ignore the capture
|
|
|
|
|
else:
|
|
|
|
|
c.ml = idx
|
|
|
|
|
of pkBackRef..pkBackRefIgnoreStyle:
|
|
|
|
|
of pkBackRef..pkBackRefIgnoreStyle:
|
|
|
|
|
if p.index >= c.ml: return -1
|
|
|
|
|
var (a, b) = c.matches[p.index]
|
|
|
|
|
var n: TPeg
|
|
|
|
|
n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef))
|
|
|
|
|
n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef))
|
|
|
|
|
n.term = s.substr(a, b)
|
|
|
|
|
result = rawMatch(s, n, start, c)
|
|
|
|
|
of pkStartAnchor:
|
|
|
|
|
@@ -769,7 +769,7 @@ proc match*(s: string, pattern: TPeg, matches: var openarray[string],
|
|
|
|
|
for i in 0..c.ml-1:
|
|
|
|
|
matches[i] = substr(s, c.matches[i][0], c.matches[i][1])
|
|
|
|
|
|
|
|
|
|
proc match*(s: string, pattern: TPeg,
|
|
|
|
|
proc match*(s: string, pattern: TPeg,
|
|
|
|
|
start = 0): bool {.rtl, extern: "npegs$1".} =
|
|
|
|
|
## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``.
|
|
|
|
|
var c: TCaptures
|
|
|
|
|
@@ -789,7 +789,7 @@ proc matchLen*(s: string, pattern: TPeg, matches: var openarray[string],
|
|
|
|
|
for i in 0..c.ml-1:
|
|
|
|
|
matches[i] = substr(s, c.matches[i][0], c.matches[i][1])
|
|
|
|
|
|
|
|
|
|
proc matchLen*(s: string, pattern: TPeg,
|
|
|
|
|
proc matchLen*(s: string, pattern: TPeg,
|
|
|
|
|
start = 0): int {.rtl, extern: "npegs$1".} =
|
|
|
|
|
## the same as ``match``, but it returns the length of the match,
|
|
|
|
|
## if there is no match, -1 is returned. Note that a match length
|
|
|
|
|
@@ -808,11 +808,11 @@ proc find*(s: string, pattern: TPeg, matches: var openarray[string],
|
|
|
|
|
if matchLen(s, pattern, matches, i) >= 0: return i
|
|
|
|
|
return -1
|
|
|
|
|
# could also use the pattern here: (!P .)* P
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc findBounds*(s: string, pattern: TPeg, matches: var openarray[string],
|
|
|
|
|
start = 0): tuple[first, last: int] {.
|
|
|
|
|
rtl, extern: "npegs$1Capture".} =
|
|
|
|
|
## returns the starting position and end position of ``pattern`` in ``s``
|
|
|
|
|
## returns the starting position and end position of ``pattern`` in ``s``
|
|
|
|
|
## and the captured
|
|
|
|
|
## substrings in the array ``matches``. If it does not match, nothing
|
|
|
|
|
## is written into ``matches`` and (-1,0) is returned.
|
|
|
|
|
@@ -820,40 +820,40 @@ proc findBounds*(s: string, pattern: TPeg, matches: var openarray[string],
|
|
|
|
|
var L = matchLen(s, pattern, matches, i)
|
|
|
|
|
if L >= 0: return (i, i+L-1)
|
|
|
|
|
return (-1, 0)
|
|
|
|
|
|
|
|
|
|
proc find*(s: string, pattern: TPeg,
|
|
|
|
|
|
|
|
|
|
proc find*(s: string, pattern: TPeg,
|
|
|
|
|
start = 0): int {.rtl, extern: "npegs$1".} =
|
|
|
|
|
## returns the starting position of ``pattern`` in ``s``. If it does not
|
|
|
|
|
## match, -1 is returned.
|
|
|
|
|
for i in start .. s.len-1:
|
|
|
|
|
if matchLen(s, pattern, i) >= 0: return i
|
|
|
|
|
return -1
|
|
|
|
|
|
|
|
|
|
iterator findAll*(s: string, pattern: TPeg, start = 0): string =
|
|
|
|
|
|
|
|
|
|
iterator findAll*(s: string, pattern: TPeg, start = 0): string =
|
|
|
|
|
## yields all matching captures of pattern in `s`.
|
|
|
|
|
var matches: array[0..MaxSubpatterns-1, string]
|
|
|
|
|
var i = start
|
|
|
|
|
while i < s.len:
|
|
|
|
|
var L = matchLen(s, pattern, matches, i)
|
|
|
|
|
if L < 0: break
|
|
|
|
|
for k in 0..MaxSubPatterns-1:
|
|
|
|
|
for k in 0..MaxSubPatterns-1:
|
|
|
|
|
if isNil(matches[k]): break
|
|
|
|
|
yield matches[k]
|
|
|
|
|
inc(i, L)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc findAll*(s: string, pattern: TPeg, start = 0): seq[string] {.
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
## returns all matching captures of pattern in `s`.
|
|
|
|
|
## If it does not match, @[] is returned.
|
|
|
|
|
accumulateResult(findAll(s, pattern, start))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template `=~`*(s: string, pattern: TPeg): expr =
|
|
|
|
|
## This calls ``match`` with an implicit declared ``matches`` array that
|
|
|
|
|
## can be used in the scope of the ``=~`` call:
|
|
|
|
|
##
|
|
|
|
|
## This calls ``match`` with an implicit declared ``matches`` array that
|
|
|
|
|
## can be used in the scope of the ``=~`` call:
|
|
|
|
|
##
|
|
|
|
|
## .. code-block:: nim
|
|
|
|
|
##
|
|
|
|
|
## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}":
|
|
|
|
|
## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}":
|
|
|
|
|
## # matches a key=value pair:
|
|
|
|
|
## echo("Key: ", matches[0])
|
|
|
|
|
## echo("Value: ", matches[1])
|
|
|
|
|
@@ -864,7 +864,7 @@ template `=~`*(s: string, pattern: TPeg): expr =
|
|
|
|
|
## echo("comment: ", matches[0])
|
|
|
|
|
## else:
|
|
|
|
|
## echo("syntax error")
|
|
|
|
|
##
|
|
|
|
|
##
|
|
|
|
|
when not declaredInScope(matches):
|
|
|
|
|
var matches {.inject.}: array[0..MaxSubpatterns-1, string]
|
|
|
|
|
match(s, pattern, matches)
|
|
|
|
|
@@ -934,10 +934,10 @@ proc replace*(s: string, sub: TPeg, by = ""): string {.
|
|
|
|
|
addf(result, by, caps)
|
|
|
|
|
inc(i, x)
|
|
|
|
|
add(result, substr(s, i))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc parallelReplace*(s: string, subs: varargs[
|
|
|
|
|
tuple[pattern: TPeg, repl: string]]): string {.
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
## Returns a modified copy of `s` with the substitutions in `subs`
|
|
|
|
|
## applied in parallel.
|
|
|
|
|
result = ""
|
|
|
|
|
@@ -954,8 +954,8 @@ proc parallelReplace*(s: string, subs: varargs[
|
|
|
|
|
add(result, s[i])
|
|
|
|
|
inc(i)
|
|
|
|
|
# copy the rest:
|
|
|
|
|
add(result, substr(s, i))
|
|
|
|
|
|
|
|
|
|
add(result, substr(s, i))
|
|
|
|
|
|
|
|
|
|
proc transformFile*(infile, outfile: string,
|
|
|
|
|
subs: varargs[tuple[pattern: TPeg, repl: string]]) {.
|
|
|
|
|
rtl, extern: "npegs$1".} =
|
|
|
|
|
@@ -972,7 +972,7 @@ proc transformFile*(infile, outfile: string,
|
|
|
|
|
quit("cannot open for writing: " & outfile)
|
|
|
|
|
else:
|
|
|
|
|
quit("cannot open for reading: " & infile)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
iterator split*(s: string, sep: TPeg): string =
|
|
|
|
|
## Splits the string `s` into substrings.
|
|
|
|
|
##
|
|
|
|
|
@@ -981,7 +981,7 @@ iterator split*(s: string, sep: TPeg): string =
|
|
|
|
|
##
|
|
|
|
|
## .. code-block:: nim
|
|
|
|
|
## for word in split("00232this02939is39an22example111", peg"\d+"):
|
|
|
|
|
## writeln(stdout, word)
|
|
|
|
|
## writeLine(stdout, word)
|
|
|
|
|
##
|
|
|
|
|
## Results in:
|
|
|
|
|
##
|
|
|
|
|
@@ -1044,14 +1044,14 @@ type
|
|
|
|
|
tkBackref, ## '$'
|
|
|
|
|
tkDollar, ## '$'
|
|
|
|
|
tkHat ## '^'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TToken {.final.} = object ## a token
|
|
|
|
|
kind: TTokKind ## the type of the token
|
|
|
|
|
modifier: TModifier
|
|
|
|
|
literal: string ## the parsed (string) literal
|
|
|
|
|
charset: set[char] ## if kind == tkCharSet
|
|
|
|
|
index: int ## if kind == tkBackref
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TPegLexer {.inheritable.} = object ## the lexer object.
|
|
|
|
|
bufpos: int ## the current position within the buffer
|
|
|
|
|
buf: cstring ## the buffer itself
|
|
|
|
|
@@ -1081,7 +1081,7 @@ proc HandleLF(L: var TPegLexer, pos: int): int =
|
|
|
|
|
result = pos+1
|
|
|
|
|
L.lineStart = result
|
|
|
|
|
|
|
|
|
|
proc init(L: var TPegLexer, input, filename: string, line = 1, col = 0) =
|
|
|
|
|
proc init(L: var TPegLexer, input, filename: string, line = 1, col = 0) =
|
|
|
|
|
L.buf = input
|
|
|
|
|
L.bufpos = 0
|
|
|
|
|
L.lineNumber = line
|
|
|
|
|
@@ -1089,69 +1089,69 @@ proc init(L: var TPegLexer, input, filename: string, line = 1, col = 0) =
|
|
|
|
|
L.lineStart = 0
|
|
|
|
|
L.filename = filename
|
|
|
|
|
|
|
|
|
|
proc getColumn(L: TPegLexer): int {.inline.} =
|
|
|
|
|
proc getColumn(L: TPegLexer): int {.inline.} =
|
|
|
|
|
result = abs(L.bufpos - L.lineStart) + L.colOffset
|
|
|
|
|
|
|
|
|
|
proc getLine(L: TPegLexer): int {.inline.} =
|
|
|
|
|
proc getLine(L: TPegLexer): int {.inline.} =
|
|
|
|
|
result = L.linenumber
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc errorStr(L: TPegLexer, msg: string, line = -1, col = -1): string =
|
|
|
|
|
var line = if line < 0: getLine(L) else: line
|
|
|
|
|
var col = if col < 0: getColumn(L) else: col
|
|
|
|
|
result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg]
|
|
|
|
|
|
|
|
|
|
proc handleHexChar(c: var TPegLexer, xi: var int) =
|
|
|
|
|
proc handleHexChar(c: var TPegLexer, xi: var int) =
|
|
|
|
|
case c.buf[c.bufpos]
|
|
|
|
|
of '0'..'9':
|
|
|
|
|
of '0'..'9':
|
|
|
|
|
xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0'))
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'a'..'f':
|
|
|
|
|
of 'a'..'f':
|
|
|
|
|
xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10)
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'A'..'F':
|
|
|
|
|
of 'A'..'F':
|
|
|
|
|
xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10)
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
else: discard
|
|
|
|
|
|
|
|
|
|
proc getEscapedChar(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
proc getEscapedChar(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
case c.buf[c.bufpos]
|
|
|
|
|
of 'r', 'R', 'c', 'C':
|
|
|
|
|
of 'r', 'R', 'c', 'C':
|
|
|
|
|
add(tok.literal, '\c')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'l', 'L':
|
|
|
|
|
of 'l', 'L':
|
|
|
|
|
add(tok.literal, '\L')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'f', 'F':
|
|
|
|
|
of 'f', 'F':
|
|
|
|
|
add(tok.literal, '\f')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'e', 'E':
|
|
|
|
|
of 'e', 'E':
|
|
|
|
|
add(tok.literal, '\e')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'a', 'A':
|
|
|
|
|
of 'a', 'A':
|
|
|
|
|
add(tok.literal, '\a')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'b', 'B':
|
|
|
|
|
of 'b', 'B':
|
|
|
|
|
add(tok.literal, '\b')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'v', 'V':
|
|
|
|
|
of 'v', 'V':
|
|
|
|
|
add(tok.literal, '\v')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 't', 'T':
|
|
|
|
|
of 't', 'T':
|
|
|
|
|
add(tok.literal, '\t')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
of 'x', 'X':
|
|
|
|
|
of 'x', 'X':
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
var xi = 0
|
|
|
|
|
handleHexChar(c, xi)
|
|
|
|
|
handleHexChar(c, xi)
|
|
|
|
|
if xi == 0: tok.kind = tkInvalid
|
|
|
|
|
else: add(tok.literal, chr(xi))
|
|
|
|
|
of '0'..'9':
|
|
|
|
|
of '0'..'9':
|
|
|
|
|
var val = ord(c.buf[c.bufpos]) - ord('0')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
var i = 1
|
|
|
|
|
while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}):
|
|
|
|
|
while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}):
|
|
|
|
|
val = val * 10 + ord(c.buf[c.bufpos]) - ord('0')
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
inc(i)
|
|
|
|
|
@@ -1164,32 +1164,32 @@ proc getEscapedChar(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
else:
|
|
|
|
|
add(tok.literal, c.buf[c.bufpos])
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
|
|
|
|
|
proc skip(c: var TPegLexer) =
|
|
|
|
|
|
|
|
|
|
proc skip(c: var TPegLexer) =
|
|
|
|
|
var pos = c.bufpos
|
|
|
|
|
var buf = c.buf
|
|
|
|
|
while true:
|
|
|
|
|
while true:
|
|
|
|
|
case buf[pos]
|
|
|
|
|
of ' ', '\t':
|
|
|
|
|
of ' ', '\t':
|
|
|
|
|
inc(pos)
|
|
|
|
|
of '#':
|
|
|
|
|
while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos)
|
|
|
|
|
of '\c':
|
|
|
|
|
pos = HandleCR(c, pos)
|
|
|
|
|
buf = c.buf
|
|
|
|
|
of '\L':
|
|
|
|
|
of '\L':
|
|
|
|
|
pos = HandleLF(c, pos)
|
|
|
|
|
buf = c.buf
|
|
|
|
|
else:
|
|
|
|
|
else:
|
|
|
|
|
break # EndOfFile also leaves the loop
|
|
|
|
|
c.bufpos = pos
|
|
|
|
|
|
|
|
|
|
proc getString(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
|
|
|
|
|
proc getString(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
tok.kind = tkStringLit
|
|
|
|
|
var pos = c.bufPos + 1
|
|
|
|
|
var buf = c.buf
|
|
|
|
|
var quote = buf[pos-1]
|
|
|
|
|
while true:
|
|
|
|
|
while true:
|
|
|
|
|
case buf[pos]
|
|
|
|
|
of '\\':
|
|
|
|
|
c.bufpos = pos
|
|
|
|
|
@@ -1200,13 +1200,13 @@ proc getString(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
break
|
|
|
|
|
elif buf[pos] == quote:
|
|
|
|
|
inc(pos)
|
|
|
|
|
break
|
|
|
|
|
break
|
|
|
|
|
else:
|
|
|
|
|
add(tok.literal, buf[pos])
|
|
|
|
|
inc(pos)
|
|
|
|
|
c.bufpos = pos
|
|
|
|
|
|
|
|
|
|
proc getDollar(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
|
|
|
|
|
proc getDollar(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
var pos = c.bufPos + 1
|
|
|
|
|
var buf = c.buf
|
|
|
|
|
if buf[pos] in {'0'..'9'}:
|
|
|
|
|
@@ -1218,8 +1218,8 @@ proc getDollar(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
else:
|
|
|
|
|
tok.kind = tkDollar
|
|
|
|
|
c.bufpos = pos
|
|
|
|
|
|
|
|
|
|
proc getCharSet(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
|
|
|
|
|
proc getCharSet(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
tok.kind = tkCharSet
|
|
|
|
|
tok.charset = {}
|
|
|
|
|
var pos = c.bufPos + 1
|
|
|
|
|
@@ -1242,7 +1242,7 @@ proc getCharSet(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
of '\C', '\L', '\0':
|
|
|
|
|
tok.kind = tkInvalid
|
|
|
|
|
break
|
|
|
|
|
else:
|
|
|
|
|
else:
|
|
|
|
|
ch = buf[pos]
|
|
|
|
|
inc(pos)
|
|
|
|
|
incl(tok.charset, ch)
|
|
|
|
|
@@ -1262,18 +1262,18 @@ proc getCharSet(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
of '\C', '\L', '\0':
|
|
|
|
|
tok.kind = tkInvalid
|
|
|
|
|
break
|
|
|
|
|
else:
|
|
|
|
|
else:
|
|
|
|
|
ch2 = buf[pos]
|
|
|
|
|
inc(pos)
|
|
|
|
|
for i in ord(ch)+1 .. ord(ch2):
|
|
|
|
|
incl(tok.charset, chr(i))
|
|
|
|
|
c.bufpos = pos
|
|
|
|
|
if caret: tok.charset = {'\1'..'\xFF'} - tok.charset
|
|
|
|
|
|
|
|
|
|
proc getSymbol(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
|
|
|
|
|
proc getSymbol(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
var pos = c.bufpos
|
|
|
|
|
var buf = c.buf
|
|
|
|
|
while true:
|
|
|
|
|
while true:
|
|
|
|
|
add(tok.literal, buf[pos])
|
|
|
|
|
inc(pos)
|
|
|
|
|
if buf[pos] notin strutils.IdentChars: break
|
|
|
|
|
@@ -1289,7 +1289,7 @@ proc getBuiltin(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
tok.kind = tkEscaped
|
|
|
|
|
getEscapedChar(c, tok) # may set tok.kind to tkInvalid
|
|
|
|
|
|
|
|
|
|
proc getTok(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
proc getTok(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
tok.kind = tkInvalid
|
|
|
|
|
tok.modifier = modNone
|
|
|
|
|
setlen(tok.literal, 0)
|
|
|
|
|
@@ -1304,11 +1304,11 @@ proc getTok(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
else:
|
|
|
|
|
tok.kind = tkCurlyLe
|
|
|
|
|
add(tok.literal, '{')
|
|
|
|
|
of '}':
|
|
|
|
|
of '}':
|
|
|
|
|
tok.kind = tkCurlyRi
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
add(tok.literal, '}')
|
|
|
|
|
of '[':
|
|
|
|
|
of '[':
|
|
|
|
|
getCharset(c, tok)
|
|
|
|
|
of '(':
|
|
|
|
|
tok.kind = tkParLe
|
|
|
|
|
@@ -1318,7 +1318,7 @@ proc getTok(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
tok.kind = tkParRi
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
add(tok.literal, ')')
|
|
|
|
|
of '.':
|
|
|
|
|
of '.':
|
|
|
|
|
tok.kind = tkAny
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
add(tok.literal, '.')
|
|
|
|
|
@@ -1326,16 +1326,16 @@ proc getTok(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
tok.kind = tkAnyRune
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
add(tok.literal, '_')
|
|
|
|
|
of '\\':
|
|
|
|
|
of '\\':
|
|
|
|
|
getBuiltin(c, tok)
|
|
|
|
|
of '\'', '"': getString(c, tok)
|
|
|
|
|
of '$': getDollar(c, tok)
|
|
|
|
|
of '\0':
|
|
|
|
|
of '\0':
|
|
|
|
|
tok.kind = tkEof
|
|
|
|
|
tok.literal = "[EOF]"
|
|
|
|
|
of 'a'..'z', 'A'..'Z', '\128'..'\255':
|
|
|
|
|
getSymbol(c, tok)
|
|
|
|
|
if c.buf[c.bufpos] in {'\'', '"'} or
|
|
|
|
|
if c.buf[c.bufpos] in {'\'', '"'} or
|
|
|
|
|
c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}:
|
|
|
|
|
case tok.literal
|
|
|
|
|
of "i": tok.modifier = modIgnoreCase
|
|
|
|
|
@@ -1383,7 +1383,7 @@ proc getTok(c: var TPegLexer, tok: var TToken) =
|
|
|
|
|
tok.kind = tkAt
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
add(tok.literal, '@')
|
|
|
|
|
if c.buf[c.bufpos] == '@':
|
|
|
|
|
if c.buf[c.bufpos] == '@':
|
|
|
|
|
tok.kind = tkCurlyAt
|
|
|
|
|
inc(c.bufpos)
|
|
|
|
|
add(tok.literal, '@')
|
|
|
|
|
@@ -1402,7 +1402,7 @@ proc arrowIsNextTok(c: TPegLexer): bool =
|
|
|
|
|
result = c.buf[pos] == '<' and c.buf[pos+1] == '-'
|
|
|
|
|
|
|
|
|
|
# ----------------------------- parser ----------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
type
|
|
|
|
|
EInvalidPeg* = object of ValueError ## raised if an invalid
|
|
|
|
|
## PEG has been detected
|
|
|
|
|
@@ -1420,7 +1420,7 @@ proc pegError(p: TPegParser, msg: string, line = -1, col = -1) =
|
|
|
|
|
e.msg = errorStr(p, msg, line, col)
|
|
|
|
|
raise e
|
|
|
|
|
|
|
|
|
|
proc getTok(p: var TPegParser) =
|
|
|
|
|
proc getTok(p: var TPegParser) =
|
|
|
|
|
getTok(p, p.tok)
|
|
|
|
|
if p.tok.kind == tkInvalid: pegError(p, "invalid token")
|
|
|
|
|
|
|
|
|
|
@@ -1470,7 +1470,7 @@ proc builtin(p: var TPegParser): TPeg =
|
|
|
|
|
of "white": result = UnicodeWhitespace()
|
|
|
|
|
else: pegError(p, "unknown built-in: " & p.tok.literal)
|
|
|
|
|
|
|
|
|
|
proc token(terminal: TPeg, p: TPegParser): TPeg =
|
|
|
|
|
proc token(terminal: TPeg, p: TPegParser): TPeg =
|
|
|
|
|
if p.skip.kind == pkEmpty: result = terminal
|
|
|
|
|
else: result = sequence(p.skip, terminal)
|
|
|
|
|
|
|
|
|
|
@@ -1491,7 +1491,7 @@ proc primary(p: var TPegParser): TPeg =
|
|
|
|
|
else: discard
|
|
|
|
|
case p.tok.kind
|
|
|
|
|
of tkIdentifier:
|
|
|
|
|
if p.identIsVerbatim:
|
|
|
|
|
if p.identIsVerbatim:
|
|
|
|
|
var m = p.tok.modifier
|
|
|
|
|
if m == modNone: m = p.modifier
|
|
|
|
|
result = modifiedTerm(p.tok.literal, m).token(p)
|
|
|
|
|
@@ -1534,17 +1534,17 @@ proc primary(p: var TPegParser): TPeg =
|
|
|
|
|
of tkEscaped:
|
|
|
|
|
result = term(p.tok.literal[0]).token(p)
|
|
|
|
|
getTok(p)
|
|
|
|
|
of tkDollar:
|
|
|
|
|
of tkDollar:
|
|
|
|
|
result = endAnchor()
|
|
|
|
|
getTok(p)
|
|
|
|
|
of tkHat:
|
|
|
|
|
of tkHat:
|
|
|
|
|
result = startAnchor()
|
|
|
|
|
getTok(p)
|
|
|
|
|
of tkBackref:
|
|
|
|
|
var m = p.tok.modifier
|
|
|
|
|
if m == modNone: m = p.modifier
|
|
|
|
|
result = modifiedBackRef(p.tok.index, m).token(p)
|
|
|
|
|
if p.tok.index < 0 or p.tok.index > p.captures:
|
|
|
|
|
if p.tok.index < 0 or p.tok.index > p.captures:
|
|
|
|
|
pegError(p, "invalid back reference index: " & $p.tok.index)
|
|
|
|
|
getTok(p)
|
|
|
|
|
else:
|
|
|
|
|
@@ -1568,7 +1568,7 @@ proc seqExpr(p: var TPegParser): TPeg =
|
|
|
|
|
while true:
|
|
|
|
|
case p.tok.kind
|
|
|
|
|
of tkAmp, tkNot, tkAt, tkStringLit, tkCharset, tkParLe, tkCurlyLe,
|
|
|
|
|
tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref,
|
|
|
|
|
tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref,
|
|
|
|
|
tkHat, tkCurlyAt:
|
|
|
|
|
result = sequence(result, primary(p))
|
|
|
|
|
of tkIdentifier:
|
|
|
|
|
@@ -1582,7 +1582,7 @@ proc parseExpr(p: var TPegParser): TPeg =
|
|
|
|
|
while p.tok.kind == tkBar:
|
|
|
|
|
getTok(p)
|
|
|
|
|
result = result / seqExpr(p)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc parseRule(p: var TPegParser): PNonTerminal =
|
|
|
|
|
if p.tok.kind == tkIdentifier and arrowIsNextTok(p):
|
|
|
|
|
result = getNonTerminal(p, p.tok.literal)
|
|
|
|
|
@@ -1596,7 +1596,7 @@ proc parseRule(p: var TPegParser): PNonTerminal =
|
|
|
|
|
incl(result.flags, ntDeclared) # NOW inlining may be attempted
|
|
|
|
|
else:
|
|
|
|
|
pegError(p, "rule expected, but found: " & p.tok.literal)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc rawParse(p: var TPegParser): TPeg =
|
|
|
|
|
## parses a rule or a PEG expression
|
|
|
|
|
while p.tok.kind == tkBuiltin:
|
|
|
|
|
@@ -1649,20 +1649,20 @@ proc peg*(pattern: string): TPeg =
|
|
|
|
|
## peg"{\ident} \s* '=' \s* {.*}"
|
|
|
|
|
result = parsePeg(pattern, "pattern")
|
|
|
|
|
|
|
|
|
|
proc escapePeg*(s: string): string =
|
|
|
|
|
proc escapePeg*(s: string): string =
|
|
|
|
|
## escapes `s` so that it is matched verbatim when used as a peg.
|
|
|
|
|
result = ""
|
|
|
|
|
var inQuote = false
|
|
|
|
|
for c in items(s):
|
|
|
|
|
for c in items(s):
|
|
|
|
|
case c
|
|
|
|
|
of '\0'..'\31', '\'', '"', '\\':
|
|
|
|
|
if inQuote:
|
|
|
|
|
of '\0'..'\31', '\'', '"', '\\':
|
|
|
|
|
if inQuote:
|
|
|
|
|
result.add('\'')
|
|
|
|
|
inQuote = false
|
|
|
|
|
result.add("\\x")
|
|
|
|
|
result.add(toHex(ord(c), 2))
|
|
|
|
|
else:
|
|
|
|
|
if not inQuote:
|
|
|
|
|
if not inQuote:
|
|
|
|
|
result.add('\'')
|
|
|
|
|
inQuote = true
|
|
|
|
|
result.add(c)
|
|
|
|
|
@@ -1675,27 +1675,27 @@ when isMainModule:
|
|
|
|
|
doAssert(not match("W_HI_L", peg"\y 'while'"))
|
|
|
|
|
doAssert(not match("W_HI_Le", peg"\y v'while'"))
|
|
|
|
|
doAssert match("W_HI_Le", peg"y'while'")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
doAssert($ +digits == $peg"\d+")
|
|
|
|
|
doAssert "0158787".match(peg"\d+")
|
|
|
|
|
doAssert "ABC 0232".match(peg"\w+\s+\d+")
|
|
|
|
|
doAssert "ABC".match(peg"\d+ / \w+")
|
|
|
|
|
|
|
|
|
|
for word in split("00232this02939is39an22example111", peg"\d+"):
|
|
|
|
|
writeln(stdout, word)
|
|
|
|
|
writeLine(stdout, word)
|
|
|
|
|
|
|
|
|
|
doAssert matchLen("key", ident) == 3
|
|
|
|
|
|
|
|
|
|
var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident)
|
|
|
|
|
doAssert matchLen("key1= cal9", pattern) == 11
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var ws = newNonTerminal("ws", 1, 1)
|
|
|
|
|
ws.rule = *whitespace
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var expr = newNonTerminal("expr", 1, 1)
|
|
|
|
|
expr.rule = sequence(capture(ident), *sequence(
|
|
|
|
|
nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var c: TCaptures
|
|
|
|
|
var s = "a+b + c +d+e+f"
|
|
|
|
|
doAssert rawMatch(s, expr.rule, 0, c) == len(s)
|
|
|
|
|
@@ -1716,7 +1716,7 @@ when isMainModule:
|
|
|
|
|
doAssert matches[0] == "abc"
|
|
|
|
|
else:
|
|
|
|
|
doAssert false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var g2 = peg"""S <- A B / C D
|
|
|
|
|
A <- 'a'+
|
|
|
|
|
B <- 'b'+
|
|
|
|
|
@@ -1733,10 +1733,10 @@ when isMainModule:
|
|
|
|
|
doAssert matches[0] == "a"
|
|
|
|
|
else:
|
|
|
|
|
doAssert false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
block:
|
|
|
|
|
var matches: array[0..2, string]
|
|
|
|
|
if match("abcdefg", peg"c {d} ef {g}", matches, 2):
|
|
|
|
|
if match("abcdefg", peg"c {d} ef {g}", matches, 2):
|
|
|
|
|
doAssert matches[0] == "d"
|
|
|
|
|
doAssert matches[1] == "g"
|
|
|
|
|
else:
|
|
|
|
|
@@ -1744,13 +1744,13 @@ when isMainModule:
|
|
|
|
|
|
|
|
|
|
for x in findAll("abcdef", peg"{.}", 3):
|
|
|
|
|
echo x
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')":
|
|
|
|
|
doAssert matches[0] == "f"
|
|
|
|
|
doAssert matches[1] == "a, b"
|
|
|
|
|
else:
|
|
|
|
|
doAssert false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
doAssert match("eine übersicht und außerdem", peg"(\letter \white*)+")
|
|
|
|
|
# ß is not a lower cased letter?!
|
|
|
|
|
doAssert match("eine übersicht und auerdem", peg"(\lower \white*)+")
|
|
|
|
|
@@ -1762,7 +1762,7 @@ when isMainModule:
|
|
|
|
|
"var1<-keykey;var2<-key2key2")
|
|
|
|
|
|
|
|
|
|
doAssert match("prefix/start", peg"^start$", 7)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# tricky test to check for false aliasing:
|
|
|
|
|
block:
|
|
|
|
|
var a = term"key"
|
|
|
|
|
|