unittests,viml/parser/expressions: Start adding asgn parsing tests

src/nvim/viml/parser/expressions.c

@@ -1363,7 +1363,6 @@ static inline ParserPosition recol_pos(const ParserPosition pos,
       } \
     } while (0)
 
-// TODO(ZyX-I): actual condition
 /// Check whether it is possible to have next expression after current
 ///
 /// For :echo: `:echo @a @a` is a valid expression. `:echo (@a @a)` is not.
@@ -1901,6 +1900,7 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags)
   bool highlighted_prev_spacing = false;
   // Lambda node, valid when parsing lambda arguments only.
   ExprASTNode *lambda_node = NULL;
+  size_t asgn_level = 0;
   do {
     const bool is_concat_or_subscript = (
         want_node == kENodeValue
@@ -2063,6 +2063,9 @@ viml_pexpr_parse_process_token:
                    && tok_type != kExprLexDot
                    && (tok_type != kExprLexComma || !is_single_assignment)
                    && tok_type != kExprLexAssignment) {
+          if (flags & kExprFlagsMulti && MAY_HAVE_NEXT_EXPR) {
+            goto viml_pexpr_parse_end;
+          }
           ERROR_FROM_TOKEN_AND_MSG(
               cur_token,
               _("E15: Expected assignment operator or subscript: %.*s"));
@@ -2429,6 +2432,10 @@ viml_pexpr_parse_valid_colon:
           ExprASTNode *new_top_node = *new_top_node_p;
           switch (new_top_node->type) {
             case kExprNodeListLiteral: {
+              if (pt_is_assignment(cur_pt) && new_top_node->children == NULL) {
+                ERROR_FROM_TOKEN_AND_MSG(
+                    cur_token, _("E475: Unable to assign to empty list: %.*s"));
+              }
               HL_CUR_TOKEN(List);
               break;
             }
@@ -2447,14 +2454,18 @@ viml_pexpr_parse_bracket_closing_error:
           }
           kvi_push(ast_stack, new_top_node_p);
           want_node = kENodeOperator;
-          if (cur_pt == kEPTSingleAssignment) {
+          if (kv_size(ast_stack) <= asgn_level) {
-            kv_drop(pt_stack, 1);
+            assert(kv_size(ast_stack) == asgn_level);
-          } else if (cur_pt == kEPTAssignment) {
+            asgn_level = 0;
-            assert(ast.err.msg);
+            if (cur_pt == kEPTSingleAssignment) {
-          } else if (cur_pt == kEPTExpr
+              kv_drop(pt_stack, 1);
-                     && kv_size(pt_stack) > 1
+            } else if (cur_pt == kEPTAssignment) {
-                     && pt_is_assignment(kv_Z(pt_stack, 1))) {
+              assert(ast.err.msg);
-            kv_drop(pt_stack, 1);
+            } else if (cur_pt == kEPTExpr
+                       && kv_size(pt_stack) > 1
+                       && pt_is_assignment(kv_Z(pt_stack, 1))) {
+              kv_drop(pt_stack, 1);
+            }
           }
         } else {
           if (want_node == kENodeValue) {
@@ -2480,6 +2491,7 @@ viml_pexpr_parse_bracket_closing_error:
             ADD_OP_NODE(cur_node);
             HL_CUR_TOKEN(SubscriptBracket);
             if (pt_is_assignment(cur_pt)) {
+              asgn_level = kv_size(ast_stack);
               kvi_push(pt_stack, kEPTExpr);
             }
           }
@@ -2586,10 +2598,14 @@ viml_pexpr_parse_figure_brace_closing_error:
           }
           kvi_push(ast_stack, new_top_node_p);
           want_node = kENodeOperator;
-          if (cur_pt == kEPTExpr
+          if (kv_size(ast_stack) <= asgn_level) {
-              && kv_size(pt_stack) > 1
+            assert(kv_size(ast_stack) == asgn_level);
-              && pt_is_assignment(kv_Z(pt_stack, 1))) {
+            if (cur_pt == kEPTExpr
-            kv_drop(pt_stack, 1);
+                && kv_size(pt_stack) > 1
+                && pt_is_assignment(kv_Z(pt_stack, 1))) {
+              kv_drop(pt_stack, 1);
+              asgn_level = 0;
+            }
           }
         } else {
           if (want_node == kENodeValue) {
@@ -2635,6 +2651,10 @@ viml_pexpr_parse_figure_brace_closing_error:
                 } while (0),
                 Curly);
           }
+          if (pt_is_assignment(cur_pt)
+              && !pt_is_assignment(kv_last(pt_stack))) {
+            asgn_level = kv_size(ast_stack);
+          }
         }
         break;
       }
@@ -2755,6 +2775,10 @@ viml_pexpr_parse_figure_brace_closing_error:
       case kExprLexDot: {
         ADD_VALUE_IF_MISSING(_("E15: Unexpected dot: %.*s"));
         if (prev_token.type == kExprLexSpacing) {
+          if (cur_pt == kEPTAssignment) {
+            ERROR_FROM_TOKEN_AND_MSG(
+                cur_token, _("E15: Cannot concatenate in assignments: %.*s"));
+          }
           NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeConcat);
           HL_CUR_TOKEN(Concat);
         } else {

test/helpers.lua

@@ -264,6 +264,9 @@ local function which(exe)
 end
 
 local function shallowcopy(orig)
+  if type(orig) ~= 'table' then
+    return orig
+  end
   local copy = {}
   for orig_key, orig_value in pairs(orig) do
     copy[orig_key] = orig_value
@@ -312,7 +315,7 @@ end
 
 -- dictdiff: find a diff so that mergedicts_copy(d1, diff) is equal to d2
 --
--- Note: does not copy values from d2
+-- Note: does not do copies of d2 values used.
 local function dictdiff(d1, d2)
   local ret = {}
   local hasdiff = false
@@ -321,7 +324,7 @@ local function dictdiff(d1, d2)
       hasdiff = true
       ret[k] = REMOVE_THIS
     elseif type(v) == type(d2[k]) then
-      if type(v) == 'table' and type(d2[k]) == 'table' then
+      if type(v) == 'table' then
         local subdiff = dictdiff(v, d2[k])
         if subdiff ~= nil then
           hasdiff = true
@@ -329,14 +332,16 @@ local function dictdiff(d1, d2)
         end
       elseif v ~= d2[k] then
         ret[k] = d2[k]
+        hasdiff = true
       end
     else
       ret[k] = d2[k]
+      hasdiff = true
     end
   end
   for k, v in pairs(d2) do
     if d1[k] == nil then
-      ret[k] = v
+      ret[k] = shallowcopy(v)
       hasdiff = true
     end
   end
@@ -406,13 +411,18 @@ format_luav = function(v, indent, opts)
   opts = opts or {}
   local linesep = '\n'
   local next_indent_arg = nil
+  local indent_shift = opts.indent_shift or '  '
+  local next_indent
+  local nl = '\n'
   if indent == nil then
     indent = ''
     linesep = ''
+    next_indent = ''
+    nl = ' '
   else
-    next_indent_arg = indent .. '  '
+    next_indent_arg = indent .. indent_shift
+    next_indent = indent .. indent_shift
   end
-  local next_indent = indent .. '  '
   local ret = ''
   if type(v) == 'string' then
     if opts.literal_strings then
@@ -430,10 +440,12 @@ format_luav = function(v, indent, opts)
     else
       local processed_keys = {}
       ret = '{' .. linesep
+      local non_empty = false
       for i, subv in ipairs(v) do
-        ret = ('%s%s%s,\n'):format(ret, next_indent,
+        ret = ('%s%s%s,%s'):format(ret, next_indent,
-                                   format_luav(subv, next_indent_arg, opts))
+                                   format_luav(subv, next_indent_arg, opts), nl)
         processed_keys[i] = true
+        non_empty = true
       end
       for k, subv in pairs(v) do
         if not processed_keys[k] then
@@ -443,9 +455,13 @@ format_luav = function(v, indent, opts)
             ret = ('%s%s[%s] = '):format(ret, next_indent,
                                          format_luav(k, nil, opts))
           end
-          ret = ret .. format_luav(subv, next_indent_arg, opts) .. ',\n'
+          ret = ret .. format_luav(subv, next_indent_arg, opts) .. ',' .. nl
+          non_empty = true
         end
       end
+      if nl == ' ' and non_empty then
+        ret = ret:sub(1, -3)
+      end
       ret = ret  .. indent .. '}'
     end
   elseif type(v) == 'number' then
@@ -495,6 +511,25 @@ local function intchar2lua(ch)
   return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch
 end
 
+local fixtbl_metatable = {
+  __newindex = function()
+    assert(false)
+  end,
+}
+
+local function fixtbl(tbl)
+  return setmetatable(tbl, fixtbl_metatable)
+end
+
+local function fixtbl_rec(tbl)
+  for k, v in pairs(tbl) do
+    if type(v) == 'table' then
+      fixtbl_rec(v)
+    end
+  end
+  return fixtbl(tbl)
+end
+
 return {
   eq = eq,
   neq = neq,
@@ -519,4 +554,6 @@ return {
   format_string = format_string,
   intchar2lua = intchar2lua,
   updated = updated,
+  fixtbl = fixtbl,
+  fixtbl_rec = fixtbl_rec,
 }

test/symbolic/klee/viml_expressions_parser.c

@@ -93,6 +93,20 @@ int main(const int argc, const char *const *const argv,
 
   const ExprAST ast = viml_pexpr_parse(&pstate, (int)flags);
   assert(ast.root != NULL || ast.err.msg);
+  if (flags & kExprFlagsParseLet) {
+    assert(ast.err.msg != NULL
+           || ast.root->type == kExprNodeAssignment
+           || (ast.root->type == kExprNodeListLiteral
+               && ast.root->children != NULL)
+           || ast.root->type == kExprNodeComplexIdentifier
+           || ast.root->type == kExprNodeCurlyBracesIdentifier
+           || ast.root->type == kExprNodePlainIdentifier
+           || ast.root->type == kExprNodeRegister
+           || ast.root->type == kExprNodeEnvironment
+           || ast.root->type == kExprNodeOption
+           || ast.root->type == kExprNodeSubscript
+           || ast.root->type == kExprNodeConcatOrSubscript);
+  }
   // Can’t possibly have more highlight tokens then there are bytes in string.
   assert(kv_size(colors) <= INPUT_SIZE - shift);
   kvi_destroy(colors);

test/unit/viml/expressions/parser_spec.lua

@@ -202,12 +202,20 @@ local function hls_to_hl_fs(hls)
   return ret
 end
 
-local function format_check(expr, format_check_data)
+local function format_check(expr, format_check_data, opts)
   -- That forces specific order.
-  local zdata = format_check_data[0]
+  local zflags = opts.flags[1]
-  print(format_string('\ncheck_parsing(%r, {', expr, flags))
+  local zdata = format_check_data[zflags]
-  local digits = '  --           '
+  local dig_len = 0
-  local digits2 = '  --  '
+  if opts.funcname then
+    print(format_string('\n%s(%r, {', opts.funcname, expr))
+    dig_len = #opts.funcname + 2
+  else
+    print(format_string('\n_check_parsing(%r, %r, {', opts, expr))
+    dig_len = #('_check_parsing(, \'') + #(format_string('%r', opts))
+  end
+  local digits = '  --' .. (' '):rep(dig_len - #('  --'))
+  local digits2 = digits:sub(1, -10)
   for i = 0, #expr - 1 do
     if i % 10 == 0 then
       digits2 = ('%s%10u'):format(digits2, i / 10)
@@ -232,12 +240,15 @@ local function format_check(expr, format_check_data)
   local diffs = {}
   local diffs_num = 0
   for flags, v in pairs(format_check_data) do
-    if flags ~= 0 then
+    if flags ~= zflags then
       diffs[flags] = dictdiff(zdata, v)
       if diffs[flags] then
-        if flags == 3 then
+        if flags == 3 + zflags then
-          if (dictdiff(format_check_data[1], format_check_data[3]) == nil
+          if (dictdiff(format_check_data[1 + zflags],
-              or dictdiff(format_check_data[2], format_check_data[3]) == nil) then
+                       format_check_data[3 + zflags]) == nil
+              or dictdiff(format_check_data[2 + zflags],
+                          format_check_data[3 + zflags]) == nil)
+          then
             diffs[flags] = nil
           else
             diffs_num = diffs_num + 1
@@ -437,10 +448,12 @@ child_call_once(function()
 end)
 
 describe('Expressions parser', function()
-  local function check_parsing(str, exp_ast, exp_highlighting_fs, nz_flags_exps)
+  local function _check_parsing(opts, str, exp_ast, exp_highlighting_fs,
+                                nz_flags_exps)
+    local zflags = opts.flags[1]
     nz_flags_exps = nz_flags_exps or {}
     local format_check_data = {}
-    for _, flags in ipairs({0, 1, 2, 3}) do
+    for _, flags in ipairs(opts.flags) do
       debug_log(('Running test case (%s, %u)'):format(str, flags))
       local err, msg = pcall(function()
         if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then
@@ -452,25 +465,25 @@ describe('Expressions parser', function()
         local east = lib.viml_pexpr_parse(pstate, flags)
         local ast = east2lua(pstate, east)
         local hls = phl2lua(pstate)
-        local exps = {
-          ast = exp_ast,
-          hl_fs = exp_highlighting_fs,
-        }
-        local add_exps = nz_flags_exps[flags]
-        if not add_exps and flags == 3 then
-          add_exps = nz_flags_exps[1] or nz_flags_exps[2]
-        end
-        if add_exps then
-          if add_exps.ast then
-            exps.ast = mergedicts_copy(exps.ast, add_exps.ast)
-          end
-          if add_exps.hl_fs then
-            exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs)
-          end
-        end
         if exp_ast == nil then
           format_check_data[flags] = {ast=ast, hl_fs=hls_to_hl_fs(hls)}
         else
+          local exps = {
+            ast = exp_ast,
+            hl_fs = exp_highlighting_fs,
+          }
+          local add_exps = nz_flags_exps[flags]
+          if not add_exps and flags == 3 + zflags then
+            add_exps = nz_flags_exps[1 + zflags] or nz_flags_exps[2 + zflags]
+          end
+          if add_exps then
+            if add_exps.ast then
+              exps.ast = mergedicts_copy(exps.ast, add_exps.ast)
+            end
+            if add_exps.hl_fs then
+              exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs)
+            end
+          end
           eq(exps.ast, ast)
           if exp_highlighting_fs then
             local exp_highlighting = {}
@@ -493,9 +506,18 @@ describe('Expressions parser', function()
       end
     end
     if exp_ast == nil then
-      format_check(str, format_check_data)
+      format_check(str, format_check_data, opts)
     end
   end
+  local function check_parsing(...)
+    return _check_parsing({flags={0, 1, 2, 3}, funcname='check_parsing'}, ...)
+  end
+  local function check_asgn_parsing(...)
+    return _check_parsing({
+      flags={4, 5, 6, 7},
+      funcname='check_asgn_parsing',
+    }, ...)
+  end
   local function hl(group, str, shift)
     return function(next_col)
       if nvim_hl_defs['NVim' .. group] == nil then
@@ -7694,6 +7716,208 @@ describe('Expressions parser', function()
       },
     })
   end)
+  itp('works with assignments', function()
+    check_asgn_parsing('a=b', {
+      --                012
+      ast = {
+        {
+          'Assignment(Plain):0:1:=',
+          children = {
+            'PlainIdentifier(scope=0,ident=a):0:0:a',
+            'PlainIdentifier(scope=0,ident=b):0:2:b',
+          },
+        },
+      },
+    }, {
+      hl('IdentifierName', 'a'),
+      hl('PlainAssignment', '='),
+      hl('IdentifierName', 'b'),
+    })
+
+    check_asgn_parsing('a+=b', {
+      --                0123
+      ast = {
+        {
+          'Assignment(Add):0:1:+=',
+          children = {
+            'PlainIdentifier(scope=0,ident=a):0:0:a',
+            'PlainIdentifier(scope=0,ident=b):0:3:b',
+          },
+        },
+      },
+    }, {
+      hl('IdentifierName', 'a'),
+      hl('AssignmentWithAddition', '+='),
+      hl('IdentifierName', 'b'),
+    })
+
+    check_asgn_parsing('a-=b', {
+      --                0123
+      ast = {
+        {
+          'Assignment(Subtract):0:1:-=',
+          children = {
+            'PlainIdentifier(scope=0,ident=a):0:0:a',
+            'PlainIdentifier(scope=0,ident=b):0:3:b',
+          },
+        },
+      },
+    }, {
+      hl('IdentifierName', 'a'),
+      hl('AssignmentWithSubtraction', '-='),
+      hl('IdentifierName', 'b'),
+    })
+
+    check_asgn_parsing('a.=b', {
+      --                0123
+      ast = {
+        {
+          'Assignment(Concat):0:1:.=',
+          children = {
+            'PlainIdentifier(scope=0,ident=a):0:0:a',
+            'PlainIdentifier(scope=0,ident=b):0:3:b',
+          },
+        },
+      },
+    }, {
+      hl('IdentifierName', 'a'),
+      hl('AssignmentWithConcatenation', '.='),
+      hl('IdentifierName', 'b'),
+    })
+
+    check_asgn_parsing('a', {
+      --                0
+      ast = {
+        'PlainIdentifier(scope=0,ident=a):0:0:a',
+      },
+    }, {
+      hl('IdentifierName', 'a'),
+    })
+
+    check_asgn_parsing('a b', {
+      --                012
+      ast = {
+        {
+          'OpMissing:0:1:',
+          children = {
+            'PlainIdentifier(scope=0,ident=a):0:0:a',
+            'PlainIdentifier(scope=0,ident=b):0:1: b',
+          },
+        },
+      },
+      err = {
+        arg = 'b',
+        msg = 'E15: Expected assignment operator or subscript: %.*s',
+      },
+    }, {
+      hl('IdentifierName', 'a'),
+      hl('InvalidSpacing', ' '),
+      hl('IdentifierName', 'b'),
+    }, {
+      [5] = {
+        ast = {
+          ast = {
+            'PlainIdentifier(scope=0,ident=a):0:0:a',
+          },
+          err = REMOVE_THIS,
+        },
+        hl_fs = {
+          [2] = REMOVE_THIS,
+          [3] = REMOVE_THIS,
+        }
+      },
+    })
+
+    check_asgn_parsing('[a, b, c]', {
+      --                012345678
+      ast = {
+        {
+          'ListLiteral:0:0:[',
+          children = {
+            {
+              'Comma:0:2:,',
+              children = {
+                'PlainIdentifier(scope=0,ident=a):0:1:a',
+                {
+                  'Comma:0:5:,',
+                  children = {
+                    'PlainIdentifier(scope=0,ident=b):0:3: b',
+                    'PlainIdentifier(scope=0,ident=c):0:6: c',
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+    }, {
+      hl('List', '['),
+      hl('IdentifierName', 'a'),
+      hl('Comma', ','),
+      hl('IdentifierName', 'b', 1),
+      hl('Comma', ','),
+      hl('IdentifierName', 'c', 1),
+      hl('List', ']'),
+    })
+
+    check_asgn_parsing('[a, b]', {
+      --                012345
+      ast = {
+        {
+          'ListLiteral:0:0:[',
+          children = {
+            {
+              'Comma:0:2:,',
+              children = {
+                'PlainIdentifier(scope=0,ident=a):0:1:a',
+                'PlainIdentifier(scope=0,ident=b):0:3: b',
+              },
+            },
+          },
+        },
+      },
+    }, {
+      hl('List', '['),
+      hl('IdentifierName', 'a'),
+      hl('Comma', ','),
+      hl('IdentifierName', 'b', 1),
+      hl('List', ']'),
+    })
+
+    check_asgn_parsing('[a]', {
+      --                012
+      ast = {
+        {
+          'ListLiteral:0:0:[',
+          children = {
+            'PlainIdentifier(scope=0,ident=a):0:1:a',
+          },
+        },
+      },
+    }, {
+      hl('List', '['),
+      hl('IdentifierName', 'a'),
+      hl('List', ']'),
+    })
+
+    check_asgn_parsing('[]', {
+      --                01
+      ast = {
+        'ListLiteral:0:0:[',
+      },
+      err = {
+        arg = ']',
+        msg = 'E475: Unable to assign to empty list: %.*s',
+      },
+    }, {
+      hl('List', '['),
+      hl('InvalidList', ']'),
+    })
+
+    -- check_asgn_parsing('a[1 + 2] += 3')
+    -- check_asgn_parsing('a[{-> {b{3}: 4}[5]}()] += 6')
+    -- check_asgn_parsing('a{1}.2[{-> {b{3}: 4}[5]}()]')
+  end)
   -- FIXME: Test assignments thoroughly
   -- FIXME: Test that parsing assignments can be used for `:for` pre-`in` part.
   -- FIXME: Somehow make functional tests use the same code. Or, at least,