diff --git a/src/nre.nim b/src/nre.nim index c519180f6d..0585e49b2f 100644 --- a/src/nre.nim +++ b/src/nre.nim @@ -47,14 +47,7 @@ from unicode import runeLenAt type Regex* = ref object ## Represents the pattern that things are matched against, constructed with - ## ``re(string, string)``. Examples: ``re"foo"``, ``re(r"foo # comment", - ## "x")``, ``re"(?x)(*ANYCRLF)foo # comment"``. For more details - ## on the leading option groups, see the `Option - ## Setting `__ - ## and the `Newline - ## Convention `__ - ## sections of the `PCRE syntax - ## manual `__. + ## ``re(string)``. Examples: ``re"foo"``, ``re(r"(*ANYCRLF)(?x)foo # comment". ## ## ``pattern: string`` ## the string that was used to create the pattern. @@ -66,33 +59,36 @@ type ## a table from the capture names to their numeric id. ## ## - ## Flags - ## ..... + ## Options + ## ....... ## - ## - ``8``, ``u``, ```` - treat both the pattern and subject as UTF8 - ## - ``9``, ```` - prevents the pattern from being interpreted as UTF, no matter - ## what - ## - ``A``, ```` - as if the pattern had a ``^`` at the beginning - ## - ``E``, ```` - DOLLAR\_ENDONLY - ## - ``f``, ```` - fails if there is not a match on the first line - ## - ``i``, ```` - case insensitive - ## - ``m``, ```` - multi-line, ``^`` and ``$`` match the beginning and end of + ## The following options may appear anywhere in the pattern, and they affect + ## the rest of it. + ## + ## - ``(?i)`` - case insensitive + ## - ``(?m)`` - multi-line: ``^`` and ``$`` match the beginning and end of ## lines, not of the subject string - ## - ``N``, ```` - turn off auto-capture, ``(?foo)`` is necessary to capture. - ## - ``s``, ```` - ``.`` matches newline - ## - ``U``, ```` - expressions are not greedy by default. ``?`` can be added to - ## a qualifier to make it greedy. - ## - ``W``, ```` - Unicode character properties; ``\w`` matches ``к``. - ## - ``X``, ```` - "Extra", character escapes without special meaning (``\w`` - ## vs. ``\a``) are errors - ## - ``x``, ```` - extended, comments (``#``) and newlines are ignored - ## (extended) - ## - ``Y``, ```` - pcre.NO\_START\_OPTIMIZE, - ## - ```` - newlines are separated by ``\r`` - ## - ```` - newlines are separated by ``\r\n`` (Windows default) - ## - ```` - newlines are separated by ``\n`` (UNIX default) - ## - ```` - newlines are separated by any of the above - ## - ```` - newlines are separated by any of the above and Unicode + ## - ``(?s)`` - ``.`` also matches newline (*dotall*) + ## - ``(?U)`` - expressions are not greedy by default. ``?`` can be added + ## to a qualifier to make it greedy + ## - ``(?x)`` - whitespace and comments (``#``) are ignored (*extended*) + ## - ``(?X)`` - character escapes without special meaning (``\w`` vs. + ## ``\a``) are errors (*extra*) + ## + ## One or a combination of these options may appear only at the beginning + ## of the pattern: + ## + ## - ``(*UTF8)`` - treat both the pattern and subject as UTF-8 + ## - ``(*UCP)`` - Unicode character properties; ``\w`` matches ``я`` + ## - ``(*U)`` - a combination of the two options above + ## - ``(*FIRSTLINE*)`` - fails if there is not a match on the first line + ## - ``(*NO_AUTO_CAPTURE)`` - turn off auto-capture for groups; + ## ``(?...)`` can be used to capture + ## - ``(*CR)`` - newlines are separated by ``\r`` + ## - ``(*LF)`` - newlines are separated by ``\n`` (UNIX default) + ## - ``(*CRLF)`` - newlines are separated by ``\r\n`` (Windows default) + ## - ``(*ANYCRLF)`` - newlines are separated by any of the above + ## - ``(*ANY)`` - newlines are separated by any of the above and Unicode ## newlines: ## ## single characters VT (vertical tab, U+000B), FF (form feed, U+000C), @@ -101,10 +97,15 @@ type ## are recognized only in UTF-8 mode. ## — man pcre ## - ## - ```` - ``\R`` matches CR, LF, or CRLF - ## - ```` - ``\R`` matches any unicode newline - ## - ```` - Javascript compatibility - ## - ```` - turn off studying; study is enabled by deafault + ## - ``(*JAVASCRIPT_COMPAT)`` - JavaScript compatibility + ## - ``(*NO_STUDY)`` - turn off studying; study is enabled by default + ## + ## For more details on the leading option groups, see the `Option + ## Setting `__ + ## and the `Newline + ## Convention `__ + ## sections of the `PCRE syntax + ## manual `__. pattern*: string ## not nil pcreObj: ptr pcre.Pcre ## not nil pcreExtra: ptr pcre.ExtraData ## nil @@ -316,71 +317,61 @@ proc `==`*(a, b: RegexMatch): bool = # Creation & Destruction {{{ # PCRE Options {{{ -let Options: Table[string, int] = { - "8" : pcre.UTF8, - "utf8" : pcre.UTF8, - "9" : pcre.NEVER_UTF, - "no_utf8" : pcre.NEVER_UTF, - "A" : pcre.ANCHORED, - "anchored" : pcre.ANCHORED, - # "C" : pcre.AUTO_CALLOUT, unsuported XXX - "E" : pcre.DOLLAR_ENDONLY, - "dollar_endonly" : pcre.DOLLAR_ENDONLY, - "f" : pcre.FIRSTLINE, - "firstline" : pcre.FIRSTLINE, - "i" : pcre.CASELESS, - "case_insensitive" : pcre.CASELESS, - "m" : pcre.MULTILINE, - "multiline" : pcre.MULTILINE, - "N" : pcre.NO_AUTO_CAPTURE, - "no_auto_capture" : pcre.NO_AUTO_CAPTURE, - "s" : pcre.DOTALL, - "dotall" : pcre.DOTALL, - "U" : pcre.UNGREEDY, - "ungreedy" : pcre.UNGREEDY, - "u" : pcre.UTF8, - "W" : pcre.UCP, - "ucp" : pcre.UCP, - "X" : pcre.EXTRA, - "extra" : pcre.EXTRA, - "x" : pcre.EXTENDED, - "extended" : pcre.EXTENDED, - "Y" : pcre.NO_START_OPTIMIZE, - "no_start_optimize" : pcre.NO_START_OPTIMIZE, - - "any" : pcre.NEWLINE_ANY, - "anycrlf" : pcre.NEWLINE_ANYCRLF, - "cr" : pcre.NEWLINE_CR, - "crlf" : pcre.NEWLINE_CRLF, - "lf" : pcre.NEWLINE_LF, - "bsr_anycrlf" : pcre.BSR_ANYCRLF, - "bsr_unicode" : pcre.BSR_UNICODE, - "js" : pcre.JAVASCRIPT_COMPAT, +const PcreOptions = { + "NEVER_UTF": pcre.NEVER_UTF, + "ANCHORED": pcre.ANCHORED, + "DOLLAR_ENDONLY": pcre.DOLLAR_ENDONLY, + "FIRSTLINE": pcre.FIRSTLINE, + "NO_AUTO_CAPTURE": pcre.NO_AUTO_CAPTURE, + "JAVASCRIPT_COMPAT": pcre.JAVASCRIPT_COMPAT, + "U": pcre.UTF8 or pcre.UCP }.toTable -proc tokenizeOptions(opts: string): tuple[flags: int, study: bool] = - result = (0, true) +# Options that are supported inside regular expressions themselves +const SkipOptions = [ + "LIMIT_MATCH=", "LIMIT_RECURSION=", "NO_AUTO_POSSESS", "NO_START_OPT", + "UTF8", "UTF16", "UTF32", "UTF", "UCP", + "CR", "LF", "CRLF", "ANYCRLF", "ANY", "BSR_ANYCRLF", "BSR_UNICODE" +] - var longOpt: string = nil - for i, c in opts: - # Handle long options {{{ - if c == '<': - longOpt = "" - continue +proc extractOptions(pattern: string): tuple[pattern: string, flags: int, study: bool] = + result = ("", 0, true) - if longOpt != nil: - if c == '>': - if longOpt == "no_study": - result.study = false - else: - result.flags = result.flags or Options.fget(longOpt) - longOpt = nil + var optionStart = 0 + var equals = false + for i, c in pattern: + if optionStart == i: + if c != '(': + break + optionStart = i + + elif optionStart == i-1: + if c != '*': + break + + elif c == ')': + let name = pattern[optionStart+2 .. i-1] + if equals or name in SkipOptions: + result.pattern.add pattern[optionStart .. i] + elif PcreOptions.hasKey name: + result.flags = result.flags or PcreOptions[name] + elif name == "NO_STUDY": + result.study = false else: - longOpt.add(c.toLower) - continue - # }}} + break + optionStart = i+1 + equals = false + + elif not equals: + if c == '=': + equals = true + if pattern[optionStart+2 .. i] notin SkipOptions: + break + elif c notin {'A'..'Z', '0'..'9', '_'}: + break + + result.pattern.add pattern[optionStart .. pattern.high] - result.flags = result.flags or Options.fget($c) # }}} type UncheckedArray {.unchecked.}[T] = array[0 .. 0, T] @@ -411,24 +402,22 @@ proc getNameToNumberTable(pattern: Regex): Table[string, int] = result[name] = num -proc initRegex(pattern: string, options: string): Regex = +proc initRegex(pattern: string, flags: int, study = true): Regex = new(result, destroyRegex) result.pattern = pattern var errorMsg: cstring var errOffset: cint - let opts = tokenizeOptions(options) - result.pcreObj = pcre.compile(cstring(pattern), # better hope int is at least 4 bytes.. - cint(opts.flags), addr errorMsg, + cint(flags), addr errorMsg, addr errOffset, nil) if result.pcreObj == nil: # failed to compile raise SyntaxError(msg: $errorMsg, pos: errOffset, pattern: pattern) - if opts.study: + if study: # XXX investigate JIT result.pcreExtra = pcre.study(result.pcreObj, 0x0, addr errorMsg) if errorMsg != nil: @@ -436,7 +425,9 @@ proc initRegex(pattern: string, options: string): Regex = result.captureNameToId = result.getNameToNumberTable() -proc re*(pattern: string, options = ""): Regex = initRegex(pattern, options) +proc re*(pattern: string): Regex = + let (pattern, flags, study) = extractOptions(pattern) + initRegex(pattern, flags, study) # }}} # Operations {{{ diff --git a/test/find.nim b/test/find.nim index 549fd6ecd9..90acaf4e1c 100644 --- a/test/find.nim +++ b/test/find.nim @@ -21,5 +21,5 @@ suite "find": check("".findAll(re"") == @[""]) check("abc".findAll(re"") == @["", "", "", ""]) check("word word".findAll(re"\b") == @["", "", "", ""]) - check("word\r\lword".findAll(re(r"$", "m")) == @["", ""]) - check("слово слово".findAll(re(r"\b", "uW")) == @["", "", "", ""]) + check("word\r\lword".findAll(re"(*ANYCRLF)(?m)$") == @["", ""]) + check("слово слово".findAll(re"(*U)\b") == @["", "", "", ""]) diff --git a/test/init.nim b/test/init.nim index 2207afad85..76ffaba19b 100644 --- a/test/init.nim +++ b/test/init.nim @@ -1,24 +1,30 @@ -import unittest -import nre +import unittest, private/pcre +include nre suite "Test NRE initialization": test "correct intialization": check(re("[0-9]+") != nil) - check(re("[0-9]+", "i") != nil) + check(re("(?i)[0-9]+") != nil) - test "correct options": - expect(SyntaxError): # ValueError would be bad - discard re("[0-9]+", - "89AEfimNsUWXxY") - expect(SyntaxError): - discard re("[0-9]+", - "" & - "" & - "") + test "options": + check(extractOptions("(*NEVER_UTF)") == + ("", pcre.NEVER_UTF, true)) + check(extractOptions("(*UTF8)(*ANCHORED)(*UCP)z") == + ("(*UTF8)(*UCP)z", pcre.ANCHORED, true)) + check(extractOptions("(*ANCHORED)(*UTF8)(*JAVASCRIPT_COMPAT)z") == + ("(*UTF8)z", pcre.ANCHORED or pcre.JAVASCRIPT_COMPAT, true)) + + check(extractOptions("(*NO_STUDY)(") == ("(", 0, false)) + + check(extractOptions("(*LIMIT_MATCH=6)(*ANCHORED)z") == + ("(*LIMIT_MATCH=6)z", pcre.ANCHORED, true)) test "incorrect options": - expect(KeyError): discard re("[0-9]+", "a") - expect(KeyError): discard re("[0-9]+", "") + for s in ["CR", "(CR", "(*CR", "(*abc)", "(*abc)CR", + "(?i)", + "(*LIMIT_MATCH=5", "(*NO_AUTO_POSSESS=5)"]: + let ss = s & "(*NEVER_UTF)" + check(extractOptions(ss) == (ss, 0, true)) test "invalid regex": expect(SyntaxError): discard re("[0-9") @@ -28,4 +34,3 @@ suite "Test NRE initialization": let ex = SyntaxError(getCurrentException()) check(ex.pos == 4) check(ex.pattern == "[0-9") - diff --git a/test/misc.nim b/test/misc.nim index 1340cdc432..f110c3f1e8 100644 --- a/test/misc.nim +++ b/test/misc.nim @@ -2,8 +2,8 @@ import unittest, nre, strutils, optional_t.nonstrict suite "Misc tests": test "unicode": - check("".find(re("", "8")).match == "") - check("перевірка".replace(re(r"\w", "uW"), "") == "") + check("".find(re"(*UTF8)").match == "") + check("перевірка".replace(re"(*U)\w", "") == "") test "empty or non-empty match": check("abc".findall(re"|.").join(":") == ":a::b::c:") diff --git a/test/split.nim b/test/split.nim index fc2bbe1b47..8064e40b7e 100644 --- a/test/split.nim +++ b/test/split.nim @@ -21,8 +21,8 @@ suite "string splitting": check("12345".split(re("")) == @["1", "2", "3", "4", "5"]) check("".split(re"") == newSeq[string]()) check("word word".split(re"\b") == @["word", " ", "word"]) - check("word\r\lword".split(re(r"$", "m")) == @["word", "\r\lword"]) - check("слово слово".split(re(r"(\b)", "uW")) == @["", "слово", "", " ", "", "слово", ""]) + check("word\r\lword".split(re"(*ANYCRLF)(?m)$") == @["word", "\r\lword"]) + check("слово слово".split(re"(*U)(\b)") == @["", "слово", "", " ", "", "слово", ""]) test "perl split tests": check("forty-two" .split(re"") .join(",") == "f,o,r,t,y,-,t,w,o")