functests: Add some more tests

src/nvim/syntax.c

@@ -6170,18 +6170,18 @@ const char *const highlight_init_cmdline[] = {
   "default link NVimInvalidDict NVimInvalidContainer",
   "default link NVimInvalidList NVimInvalidContainer",
 
-  "default link NVimInvalidIdentifier Identifier",
+  "default link NVimInvalidValue NVimInvalid",
-  "default link NVimInvalidIdentifierScope NVimIdentifier",
+
-  "default link NVimInvalidIdentifierScopeDelimiter NVimIdentifier",
+  "default link NVimInvalidIdentifier NVimInvalidValue",
-  "default link NVimInvalidIdentifierName NVimIdentifier",
+  "default link NVimInvalidIdentifierScope NVimInvalidIdentifier",
-  "default link NVimInvalidIdentifierKey NVimIdentifier",
+  "default link NVimInvalidIdentifierScopeDelimiter NVimInvalidIdentifier",
+  "default link NVimInvalidIdentifierName NVimInvalidIdentifier",
+  "default link NVimInvalidIdentifierKey NVimInvalidIdentifier",
 
   "default link NVimInvalidColon NVimInvalidDelimiter",
   "default link NVimInvalidComma NVimInvalidDelimiter",
   "default link NVimInvalidArrow NVimInvalidDelimiter",
 
-  "default link NVimInvalidValue NVimInvalid",
-
   "default link NVimInvalidRegister NVimInvalidValue",
   "default link NVimInvalidNumber NVimInvalidValue",
   "default link NVimInvalidFloat NVimInvalidNumber",
@@ -6199,7 +6199,7 @@ const char *const highlight_init_cmdline[] = {
   // Invalid string bodies and specials are still highlighted as valid ones to
   // minimize the red area.
   "default link NVimInvalidString NVimInvalidValue",
-  "default link NVimInvalidStringBody NVimString",
+  "default link NVimInvalidStringBody NVimStringBody",
   "default link NVimInvalidStringQuote NVimInvalidString",
   "default link NVimInvalidStringSpecial NVimStringSpecial",
 

test/functional/ui/cmdline_highlight_spec.lua

@@ -147,6 +147,10 @@ before_each(function()
     PE={bold = true, foreground = Screen.colors.SeaGreen4},
     NUM={foreground = Screen.colors.Blue2},
     NPAR={foreground = Screen.colors.Yellow},
+    SQ={foreground = Screen.colors.Blue3},
+    SB={foreground = Screen.colors.Blue4},
+    E={foreground = Screen.colors.Red, background = Screen.colors.Blue},
+    M={bold = true},
   })
 end)
 
@@ -898,8 +902,83 @@ describe('Expressions coloring support', function()
       ={NUM:1}^                                      |
     ]])
   end)
-  -- FIXME: Test expr coloring when using -u NORC and -u NONE.
+  it('works correctly with non-ASCII and control characters', function()
-  -- FIXME: Test different ways of triggering expression highlighting (:<C-r>=,
+    meths.command('hi clear NVimStringBody')
-  --        i<C-r>=, :<C-\>e, "=).
+    meths.command('hi clear NVimStringQuote')
-  -- FIXME: Test with various invalid unicode and multibyte characters.
+    meths.command('hi clear NVimInvalid')
+    meths.command('hi NVimStringQuote guifg=Blue3')
+    meths.command('hi NVimStringBody guifg=Blue4')
+    meths.command('hi NVimInvalid guifg=Red guibg=Blue')
+    feed('i<C-r>="«»"«»')
+    screen:expect([[
+                                              |
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      ={SQ:"}{SB:«»}{SQ:"}{E:«»}^                                 |
+    ]])
+    feed('<C-c>')
+    screen:expect([[
+      ^                                        |
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {M:-- INSERT --}                            |
+    ]])
+    feed('<Esc>')
+    screen:expect([[
+      ^                                        |
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+                                              |
+    ]])
+    feed(':<C-\\>e"<C-v><C-x>"<C-v><C-x>')
+    -- TODO(ZyX-I): Parser highlighting should not override special character
+    --              highlighting.
+    screen:expect([[
+                                              |
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      ={SQ:"}{SB:^X}{SQ:"}{ERR:^X}^                                 |
+    ]])
+    feed('<C-c>')
+    screen:expect([[
+                                              |
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      :^                                       |
+    ]])
+    funcs.setreg('a', {'\192'})
+    feed('<C-r>="<C-r><C-r>a"<C-r><C-r>a"foo"')
+    -- TODO(ZyX-I): Parser highlighting should not override special character
+    --              highlighting.
+    screen:expect([[
+                                              |
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      ={SQ:"}{SB:<c0>}{SQ:"}{E:<c0>"}{SB:foo}{E:"}^                        |
+    ]])
+  end)
 end)

test/unit/viml/expressions/parser_tests.lua

@@ -8182,4 +8182,92 @@ return function(itp, _check_parsing, hl, fmtn)
       hl('List', ']'),
     })
   end)
+  itp('works with non-ASCII characters', function()
+    check_parsing('"«»"«»', {
+      --           013568
+      ast = {
+        {
+          'OpMissing:0:6:',
+          children = {
+            'DoubleQuotedString(val="«»"):0:0:"«»"',
+            {
+              'ComplexIdentifier:0:8:',
+              children = {
+                'PlainIdentifier(scope=0,ident=«):0:6:«',
+                'PlainIdentifier(scope=0,ident=»):0:8:»',
+              },
+            },
+          },
+        },
+      },
+      err = {
+        arg = '«»',
+        msg = 'E15: Unidentified character: %.*s',
+      },
+    }, {
+      hl('DoubleQuote', '"'),
+      hl('DoubleQuotedBody', '«»'),
+      hl('DoubleQuote', '"'),
+      hl('InvalidIdentifierName', '«'),
+      hl('InvalidIdentifierName', '»'),
+    }, {
+      [1] = {
+        ast = {
+          ast = {
+            'DoubleQuotedString(val="«»"):0:0:"«»"',
+          },
+          len = 6,
+        },
+        hl_fs = {
+          [5] = REMOVE_THIS,
+          [4] = REMOVE_THIS,
+        },
+      },
+    })
+    check_parsing('"\192"\192"foo"', {
+      --           01   23   45678
+      ast = {
+        {
+          'OpMissing:0:3:',
+          children = {
+            'DoubleQuotedString(val="\192"):0:0:"\192"',
+            {
+              'OpMissing:0:4:',
+              children = {
+                'PlainIdentifier(scope=0,ident=\192):0:3:\192',
+                'DoubleQuotedString(val="foo"):0:4:"foo"',
+              },
+            },
+          },
+        },
+      },
+      err = {
+        arg = '\192"foo"',
+        msg = 'E15: Unidentified character: %.*s',
+      },
+    }, {
+      hl('DoubleQuote', '"'),
+      hl('DoubleQuotedBody', '\192'),
+      hl('DoubleQuote', '"'),
+      hl('InvalidIdentifierName', '\192'),
+      hl('InvalidDoubleQuote', '"'),
+      hl('InvalidDoubleQuotedBody', 'foo'),
+      hl('InvalidDoubleQuote', '"'),
+    }, {
+      [1] = {
+        ast = {
+          ast = {
+            'DoubleQuotedString(val="\192"):0:0:"\192"',
+          },
+          len = 3,
+        },
+        hl_fs = {
+          [4] = REMOVE_THIS,
+          [5] = REMOVE_THIS,
+          [6] = REMOVE_THIS,
+          [7] = REMOVE_THIS,
+        },
+      },
+    })
+  end)
 end