Merge pull request #7907 from GULPF/lexer-unicode-fix

Lexer fix for multi byte characters

compiler/lexer.nim

@@ -25,7 +25,7 @@ const
   SymChars*: set[char] = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}
   SymStartChars*: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
   OpChars*: set[char] = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
-    '|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'}
+    '|', '=', '%', '&', '$', '@', '~', ':'}
 
 # don't forget to update the 'highlite' module if these charsets should change
 

lib/packages/docutils/highlite.nim

@@ -130,7 +130,7 @@ proc nimNumber(g: var GeneralTokenizer, position: int): int =
 
 const
   OpChars  = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
-              '|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'}
+              '|', '=', '%', '&', '$', '@', '~', ':'}
 
 proc nimNextToken(g: var GeneralTokenizer) =
   const

tests/lexer/thexlit.nim

@@ -1,12 +0,0 @@
-discard """
-  file: "thexlit.nim"
-  output: "equal"
-"""
-
-var t=0x950412DE
-
-if t==0x950412DE:
-    echo "equal"
-else:
-    echo "not equal"
-

tests/lexer/thexrange.nim

@@ -1,8 +0,0 @@
-
-type
-  TArray = array[0x0012..0x0013, int]
-
-var a: TArray
-
-echo a[0x0012] #OUT 0
-

tests/lexer/tlexermisc.nim

@@ -0,0 +1,27 @@
+discard """
+  action: run
+  output: "equal"
+"""
+
+var t=0x950412DE
+
+if t==0x950412DE:
+    echo "equal"
+else:
+    echo "not equal"
+
+type
+  TArray = array[0x0012..0x0013, int]
+
+var a: TArray
+
+doAssert a[0x0012] == 0
+
+
+# #7884
+
+type Obj = object
+    ö: int
+
+let o = Obj(ö: 1)
+doAssert o.ö == 1