mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-06-06 11:54:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixes edge cases in the lexer

Browse Source
This commit is contained in:
Andreas Rumpf
2016-07-12 01:05:52 +02:00
parent 7a018007a4
commit 56f99f56ac
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
compiler/lexer.nim
View File

@@ -735,7 +735,8 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
if c == '\226' and
buf[pos+1] == '\128' and
buf[pos+2] == '\147': # It's a 'magic separator' en-dash Unicode
if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars:
if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars or
isMagicIdentSeparatorRune(buf, pos+magicIdentSeparatorRuneByteWidth) or pos == L.bufpos:
lexMessage(L, errInvalidToken, "–")
break
inc(pos, magicIdentSeparatorRuneByteWidth)
@@ -747,7 +748,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
h = h !& ord(c)
inc(pos)
of '_':
if buf[pos+1] notin SymChars:
if buf[pos+1] notin SymChars or isMagicIdentSeparatorRune(buf, pos+1):
lexMessage(L, errInvalidToken, "_")
break
inc(pos)
@@ -1056,7 +1057,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
inc(L.bufpos)
of '_':
inc(L.bufpos)
if L.buf[L.bufpos] notin SymChars:
if L.buf[L.bufpos] notin SymChars+{'_'} and not
isMagicIdentSeparatorRune(L.buf, L.bufpos):
tok.tokType = tkSymbol
tok.ident = getIdent("_")
else: