diff --git a/changelog.md b/changelog.md index f87dadae63..af91b49773 100644 --- a/changelog.md +++ b/changelog.md @@ -81,6 +81,9 @@ parameter and result types, not just their source-level shape. Use - `std/re` and `std/nre` are deprecated as PCRE library is obsolete. Use https://github.com/nitely/nim-regex or `std/nre2`. See: https://github.com/nim-lang/Nim/issues/23668. +- `std/pegs` now correctly lexes UTF-8 bytes inside bare identifier-style + terminals, so case-insensitive matching of non-ASCII terms (e.g. ``\i café``) + works without single-quoting. ## Language changes diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 97d586a7c1..eaece6fa12 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1668,7 +1668,10 @@ func getSymbol(c: var PegLexer, tok: var Token) = while pos < c.buf.len: add(tok.literal, c.buf[pos]) inc(pos) - if pos < c.buf.len and c.buf[pos] notin strutils.IdentChars: break + if pos < c.buf.len: + let ch = c.buf[pos] + # Keep non-ASCII bytes so UTF-8 terminals reach the rune-aware matchers. + if ch notin strutils.IdentChars and ord(ch) < 0x80: break c.bufpos = pos tok.kind = tkIdentifier diff --git a/tests/stdlib/tpegs.nim b/tests/stdlib/tpegs.nim index 18753a9bea..04774aa35b 100644 --- a/tests/stdlib/tpegs.nim +++ b/tests/stdlib/tpegs.nim @@ -259,6 +259,11 @@ block: doAssert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") doAssert(not match("456678", peg"(\letter)+")) + block: + doAssert match("CAFÉ", peg"\i café") + doAssert match("Café", peg"\i café") + doAssert "two cafés: Café and CAFÉ".findAll(peg"\i café").len == 3 + doAssert("var1 = key; var2 = key2".replacef( peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == "var1<-keykey;var2<-key2key2")