test/util: expect_msg_seq()

job_spec.lua on AppVeyor (Windows) often fails like this:

      FAILED  ] C:/projects/neovim/test/functional\core\job_spec.lua @ 72: jobs changes to given `cwd` directory
    C:/projects/neovim/test/functional\core\job_spec.lua:81: Expected objects to be the same.
    Passed in:
    (table) {
      [1] = 'notification'
      [2] = 'stdout'
     *[3] = {
        [1] = 0
       *[2] = {
          [1] = 'C:\projects\neovim\Xtest-tmpdir\nvimmSjq1S\0' } } }
    Expected:
    (table) {
      [1] = 'notification'
      [2] = 'stdout'
     *[3] = {
        [1] = 0
       *[2] = {
          [1] = 'C:\projects\neovim\Xtest-tmpdir\nvimmSjq1S\0'
         *[2] = '' } } }
    stack traceback:

Message chunking is non-deterministic, so we need to try different
variants.

test/functional/core/job_spec.lua

@@ -12,6 +12,7 @@ local get_pathsep = helpers.get_pathsep
 local pathroot = helpers.pathroot
 local nvim_set = helpers.nvim_set
 local expect_twostreams = helpers.expect_twostreams
+local expect_msg_seq = helpers.expect_msg_seq
 local Screen = require('test.functional.ui.screen')
 
 describe('jobs', function()
@@ -78,9 +79,18 @@ describe('jobs', function()
     else
       nvim('command', "let j = jobstart('pwd', g:job_opts)")
     end
-    eq({'notification', 'stdout', {0, {dir, ''}}}, next_msg())
+    expect_msg_seq(
-    eq({'notification', 'stdout', {0, {''}}}, next_msg())
+      { {'notification', 'stdout', {0, {dir, ''} } },
-    eq({'notification', 'exit', {0, 0}}, next_msg())
+        {'notification', 'stdout', {0, {''} } },
+        {'notification', 'exit', {0, 0} }
+      },
+      -- Alternative sequence:
+      { {'notification', 'stdout', {0, {dir} } },
+        {'notification', 'stdout', {0, {'', ''} } },
+        {'notification', 'stdout', {0, {''} } },
+        {'notification', 'exit', {0, 0} }
+      }
+    )
     rmdir(dir)
   end)
 
@@ -308,8 +318,15 @@ describe('jobs', function()
     nvim('command', 'unlet g:job_opts.on_exit')
     nvim('command', 'let g:job_opts.user = 5')
     nvim('command', [[call jobstart('echo "foo"', g:job_opts)]])
-    eq({'notification', 'stdout', {5, {'foo', ''}}}, next_msg())
+    expect_msg_seq(
-    eq({'notification', 'stdout', {5, {''}}}, next_msg())
+      { {'notification', 'stdout', {5, {'foo', ''} } },
+        {'notification', 'stdout', {5, {''} } }
+      },
+      -- Alternative sequence:
+      { {'notification', 'stdout', {5, {'foo'} } },
+        {'notification', 'stdout', {5, {'', ''} } }
+      }
+    )
   end)
 
   it('will pass return code with the exit event', function()

test/functional/helpers.lua

@@ -14,10 +14,12 @@ local check_cores = global_helpers.check_cores
 local check_logs = global_helpers.check_logs
 local neq = global_helpers.neq
 local eq = global_helpers.eq
+local eq_any = global_helpers.eq_any
 local ok = global_helpers.ok
 local map = global_helpers.map
 local filter = global_helpers.filter
 local dedent = global_helpers.dedent
+local table_flatten = global_helpers.table_flatten
 
 local start_dir = lfs.currentdir()
 -- XXX: NVIM_PROG takes precedence, QuickBuild sets it.
@@ -96,8 +98,8 @@ local function request(method, ...)
   return rv
 end
 
-local function next_message()
+local function next_message(timeout)
-  return session:next_message()
+  return session:next_message(timeout)
 end
 
 local function expect_twostreams(msgs1, msgs2)
@@ -116,6 +118,46 @@ local function expect_twostreams(msgs1, msgs2)
   end
 end
 
+-- Expects a sequence of next_message() results. If multiple sequences are
+-- passed they are tried until one succeeds, in order of shortest to longest.
+local function expect_msg_seq(...)
+  if select('#', ...) < 1 then
+    error('need at least 1 argument')
+  end
+  local seqs = {...}
+  table.sort(seqs, function(a, b)  -- Sort ascending, by (shallow) length.
+    return #a < #b
+  end)
+
+  local actual_seq = {}
+  local final_error = ''
+  local function cat_err(err1, err2)
+    if err1 == nil then
+      return err2
+    end
+    return string.format('%s\n%s\n%s', err1, string.rep('=', 78), err2)
+  end
+  for anum = 1, #seqs do
+    local expected_seq = seqs[anum]
+    -- Collect enough messages to compare the next expected sequence.
+    while #actual_seq < #expected_seq do
+      local msg = next_message(10000)  -- Big timeout for ASAN/valgrind.
+      if msg == nil then
+        error(cat_err(final_error,
+                      string.format('got %d messages, expected %d',
+                                    #actual_seq, #expected_seq)))
+      end
+      table.insert(actual_seq, msg)
+    end
+    local status, result = pcall(eq, expected_seq, actual_seq)
+    if status then
+      return result
+    end
+    final_error = cat_err(final_error, result)
+  end
+  error(final_error)
+end
+
 local function call_and_stop_on_error(...)
   local status, result = copcall(...)  -- luacheck: ignore
   if not status then
@@ -674,78 +716,81 @@ local function hexdump(str)
 end
 
 local module = {
-  prepend_argv = prepend_argv,
+  NIL = mpack.NIL,
-  clear = clear,
+  alter_slashes = alter_slashes,
-  connect = connect,
+  buffer = buffer,
-  retry = retry,
+  bufmeths = bufmeths,
-  spawn = spawn,
-  dedent = dedent,
-  source = source,
-  rawfeed = rawfeed,
-  insert = insert,
-  iswin = iswin,
-  feed = feed,
-  feed_command = feed_command,
-  eval = nvim_eval,
   call = nvim_call,
+  clear = clear,
   command = nvim_command,
-  request = request,
+  connect = connect,
-  next_message = next_message,
+  curbuf = curbuf,
-  expect_twostreams = expect_twostreams,
+  curbuf_contents = curbuf_contents,
-  run = run,
+  curbufmeths = curbufmeths,
-  stop = stop,
+  curtab = curtab,
+  curtabmeths = curtabmeths,
+  curwin = curwin,
+  curwinmeths = curwinmeths,
+  dedent = dedent,
   eq = eq,
-  neq = neq,
+  eq_any = eq_any,
+  eval = nvim_eval,
+  exc_exec = exc_exec,
   expect = expect,
   expect_any = expect_any,
-  ok = ok,
+  expect_msg_seq = expect_msg_seq,
-  map = map,
+  expect_twostreams = expect_twostreams,
+  feed = feed,
+  feed_command = feed_command,
   filter = filter,
-  nvim = nvim,
-  nvim_async = nvim_async,
-  nvim_prog = nvim_prog,
-  nvim_argv = nvim_argv,
-  nvim_set = nvim_set,
-  nvim_dir = nvim_dir,
-  buffer = buffer,
-  window = window,
-  tabpage = tabpage,
-  curbuf = curbuf,
-  curwin = curwin,
-  curtab = curtab,
-  curbuf_contents = curbuf_contents,
-  wait = wait,
-  sleep = sleep,
-  set_session = set_session,
-  write_file = write_file,
-  read_file = read_file,
-  os_name = os_name,
-  rmdir = rmdir,
-  mkdir = lfs.mkdir,
-  exc_exec = exc_exec,
-  redir_exec = redir_exec,
-  merge_args = merge_args,
   funcs = funcs,
-  meths = meths,
-  bufmeths = bufmeths,
-  winmeths = winmeths,
-  tabmeths = tabmeths,
-  uimeths = uimeths,
-  curbufmeths = curbufmeths,
-  curwinmeths = curwinmeths,
-  curtabmeths = curtabmeths,
-  pending_win32 = pending_win32,
-  skip_fragile = skip_fragile,
-  set_shell_powershell = set_shell_powershell,
-  tmpname = tmpname,
-  meth_pcall = meth_pcall,
-  NIL = mpack.NIL,
   get_pathsep = get_pathsep,
-  pathroot = pathroot,
-  missing_provider = missing_provider,
-  alter_slashes = alter_slashes,
   hexdump = hexdump,
+  insert = insert,
+  iswin = iswin,
+  map = map,
+  merge_args = merge_args,
+  meth_pcall = meth_pcall,
+  meths = meths,
+  missing_provider = missing_provider,
+  mkdir = lfs.mkdir,
+  neq = neq,
   new_pipename = new_pipename,
+  next_message = next_message,
+  nvim = nvim,
+  nvim_argv = nvim_argv,
+  nvim_async = nvim_async,
+  nvim_dir = nvim_dir,
+  nvim_prog = nvim_prog,
+  nvim_set = nvim_set,
+  ok = ok,
+  os_name = os_name,
+  pathroot = pathroot,
+  pending_win32 = pending_win32,
+  prepend_argv = prepend_argv,
+  rawfeed = rawfeed,
+  read_file = read_file,
+  redir_exec = redir_exec,
+  request = request,
+  retry = retry,
+  rmdir = rmdir,
+  run = run,
+  set_session = set_session,
+  set_shell_powershell = set_shell_powershell,
+  skip_fragile = skip_fragile,
+  sleep = sleep,
+  source = source,
+  spawn = spawn,
+  stop = stop,
+  table_flatten = table_flatten,
+  tabmeths = tabmeths,
+  tabpage = tabpage,
+  tmpname = tmpname,
+  uimeths = uimeths,
+  wait = wait,
+  window = window,
+  winmeths = winmeths,
+  write_file = write_file,
 }
 
 return function(after_each)

test/helpers.lua

@@ -7,13 +7,33 @@ local check_logs_useless_lines = {
   ['See README_MISSING_SYSCALL_OR_IOCTL for guidance']=3,
 }
 
-local eq = function(exp, act)
+local function eq(expected, actual)
-  return assert.are.same(exp, act)
+  return assert.are.same(expected, actual)
 end
-local neq = function(exp, act)
+-- Tries multiple accepted ("expected") values until one succeeds.
+-- First N-1 arguments are the accepted values.
+-- Last (Nth) argument is the "actual" value to be compared.
+local function eq_any(...)
+  if select('#', ...) < 2 then
+    error('need at least 2 arguments')
+  end
+  local args = {...}
+  local actual = args[#args]
+  local final_error = ''
+  for anum = 1, select('#', ...) - 1 do
+    local accepted = args[anum]
+    local status, result = pcall(eq, accepted, actual)
+    if status then
+      return result
+    end
+    final_error = final_error..tostring(result)
+  end
+  error(final_error)
+end
+local function neq(exp, act)
   return assert.are_not.same(exp, act)
 end
-local ok = function(res)
+local function ok(res)
   return assert.is_true(res)
 end
 
@@ -534,30 +554,50 @@ local function fixtbl_rec(tbl)
   return fixtbl(tbl)
 end
 
+-- From https://github.com/premake/premake-core/blob/master/src/base/table.lua
+local function table_flatten(arr)
+  local result = {}
+  local function _table_flatten(_arr)
+    local n = #_arr
+    for i = 1, n do
+      local v = _arr[i]
+      if type(v) == "table" then
+        _table_flatten(v)
+      elseif v then
+        table.insert(result, v)
+      end
+    end
+  end
+  _table_flatten(arr)
+  return result
+end
+
 return {
-  eq = eq,
-  neq = neq,
-  ok = ok,
-  check_logs = check_logs,
-  uname = uname,
-  tmpname = tmpname,
-  map = map,
-  filter = filter,
-  glob = glob,
-  check_cores = check_cores,
-  hasenv = hasenv,
-  which = which,
-  shallowcopy = shallowcopy,
-  deepcopy = deepcopy,
-  mergedicts_copy = mergedicts_copy,
-  dictdiff = dictdiff,
   REMOVE_THIS = REMOVE_THIS,
+  check_cores = check_cores,
+  check_logs = check_logs,
   concat_tables = concat_tables,
   dedent = dedent,
-  format_luav = format_luav,
+  deepcopy = deepcopy,
-  format_string = format_string,
+  dictdiff = dictdiff,
-  intchar2lua = intchar2lua,
+  eq = eq,
-  updated = updated,
+  eq_any = eq_any,
+  filter = filter,
   fixtbl = fixtbl,
   fixtbl_rec = fixtbl_rec,
+  format_luav = format_luav,
+  format_string = format_string,
+  glob = glob,
+  hasenv = hasenv,
+  intchar2lua = intchar2lua,
+  map = map,
+  mergedicts_copy = mergedicts_copy,
+  neq = neq,
+  ok = ok,
+  shallowcopy = shallowcopy,
+  table_flatten = table_flatten,
+  tmpname = tmpname,
+  uname = uname,
+  updated = updated,
+  which = which,
 }