Merge #6418 from ZyX-I/better-unittests

test/README.md

@@ -27,8 +27,8 @@ groups by the semantic component they are testing.
 
 Test behaviour is affected by environment variables. Currently supported 
 (Functional, Unit, Benchmarks) (when Defined; when set to _1_; when defined, 
-treated as Integer; when defined, treated as String; !must be defined to 
-function properly):
+treated as Integer; when defined, treated as String; when defined, treated as 
+Number; !must be defined to function properly):
 
 `GDB` (F) (D): makes nvim instances to be run under `gdbserver`. It will be 
 accessible on `localhost:7777`: use `gdb build/bin/nvim`, type `target remote 
@@ -99,3 +99,12 @@ get backtrace from).
 approximately 90% of the tests. Should be used when finding cores is too hard 
 for some reason. Normally (on OS X or when `NVIM_TEST_CORE_GLOB_DIRECTORY` is 
 defined and this variable is not) cores are checked for after each test.
+
+`NVIM_TEST_RUN_TESTTEST` (U) (1): allows running `test/unit/testtest_spec.lua` 
+used to check how testing infrastructure works.
+
+`NVIM_TEST_TRACE_LEVEL` (U) (N): specifies unit tests tracing level: `0` 
+disables tracing (the fastest, but you get no data if tests crash and there was 
+no core dump generated), `1` or empty/undefined leaves only C function cals and 
+returns in the trace (faster then recording everything), `2` records all 
+function calls, returns and lua source lines exuecuted.

test/functional/helpers.lua

@@ -16,6 +16,7 @@ local eq = global_helpers.eq
 local ok = global_helpers.ok
 local map = global_helpers.map
 local filter = global_helpers.filter
+local dedent = global_helpers.dedent
 
 local start_dir = lfs.currentdir()
 -- XXX: NVIM_PROG takes precedence, QuickBuild sets it.
@@ -191,28 +192,6 @@ local function nvim_feed(input)
   end
 end
 
-local function dedent(str)
-  -- find minimum common indent across lines
-  local indent = nil
-  for line in str:gmatch('[^\n]+') do
-    local line_indent = line:match('^%s+') or ''
-    if indent == nil or #line_indent < #indent then
-      indent = line_indent
-    end
-  end
-  if indent == nil or #indent == 0 then
-    -- no minimum common indent
-    return str
-  end
-  -- create a pattern for the indent
-  indent = indent:gsub('%s', '[ \t]')
-  -- strip it from the first line
-  str = str:gsub('^'..indent, '')
-  -- strip it from the remaining lines
-  str = str:gsub('[\n]'..indent, '\n')
-  return str
-end
-
 local function feed(...)
   for _, v in ipairs({...}) do
     nvim_feed(dedent(v))

test/helpers.lua

@@ -251,6 +251,28 @@ local function concat_tables(...)
   return ret
 end
 
+local function dedent(str)
+  -- find minimum common indent across lines
+  local indent = nil
+  for line in str:gmatch('[^\n]+') do
+    local line_indent = line:match('^%s+') or ''
+    if indent == nil or #line_indent < #indent then
+      indent = line_indent
+    end
+  end
+  if indent == nil or #indent == 0 then
+    -- no minimum common indent
+    return str
+  end
+  -- create a pattern for the indent
+  indent = indent:gsub('%s', '[ \t]')
+  -- strip it from the first line
+  str = str:gsub('^'..indent, '')
+  -- strip it from the remaining lines
+  str = str:gsub('[\n]'..indent, '\n')
+  return str
+end
+
 return {
   eq = eq,
   neq = neq,
@@ -265,4 +287,5 @@ return {
   hasenv = hasenv,
   which = which,
   concat_tables = concat_tables,
+  dedent = dedent,
 }

test/unit/helpers.lua

@@ -11,6 +11,7 @@ local posix = nil
 local syscall = nil
 
 local check_cores = global_helpers.check_cores
+local dedent = global_helpers.dedent
 local neq = global_helpers.neq
 local map = global_helpers.map
 local eq = global_helpers.eq
@@ -511,6 +512,189 @@ if os.getenv('NVIM_TEST_PRINT_SYSCALLS') == '1' then
   end
 end
 
+local function just_fail(_)
+  return false
+end
+say:set('assertion.just_fail.positive', '%s')
+say:set('assertion.just_fail.negative', '%s')
+assert:register('assertion', 'just_fail', just_fail,
+                'assertion.just_fail.positive',
+                'assertion.just_fail.negative')
+
+local hook_fnamelen = 30
+local hook_sfnamelen = 30
+local hook_numlen = 5
+local hook_msglen = 1 + 1 + 1 + (1 + hook_fnamelen) + (1 + hook_sfnamelen) + (1 + hook_numlen) + 1
+
+local tracehelp = dedent([[
+  ┌ Trace type: _r_eturn from function , function _c_all, _l_ine executed,
+  │             _t_ail return, _C_ount (should not actually appear).
+  │┏ Function type: _L_ua function, _C_ function, _m_ain part of chunk,
+  │┃                function that did _t_ail call.
+  │┃┌ Function name type: _g_lobal, _l_ocal, _m_ethod, _f_ield, _u_pvalue,
+  │┃│                     space for unknown.
+  │┃│ ┏ Source file name             ┌ Function name                ┏ Line
+  │┃│ ┃ (trunc to 30 bytes, no .lua) │ (truncated to last 30 bytes) ┃ number
+  CWN SSSSSSSSSSSSSSSSSSSSSSSSSSSSSS:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF:LLLLL\n
+]])
+
+local function child_sethook(wr)
+  local trace_level = os.getenv('NVIM_TEST_TRACE_LEVEL')
+  if not trace_level or trace_level == '' then
+    trace_level = 1
+  else
+    trace_level = tonumber(trace_level)
+  end
+  if trace_level <= 0 then
+    return
+  end
+  local trace_only_c = trace_level <= 1
+  local function hook(reason, lnum)
+    local info = nil
+    if reason ~= 'tail return' then  -- tail return
+      info = debug.getinfo(2, 'nSl')
+    end
+
+    if trace_only_c and (not info or info.what ~= 'C') then
+      return
+    end
+
+    local whatchar = ' '
+    local namewhatchar = ' '
+    local funcname = ''
+    local source = ''
+    local msgchar = reason:sub(1, 1)
+
+    if reason == 'count' then
+      msgchar = 'C'
+    end
+
+    if info then
+      funcname = (info.name or ''):sub(1, hook_fnamelen)
+      whatchar = info.what:sub(1, 1)
+      namewhatchar = info.namewhat:sub(1, 1)
+      if namewhatchar == '' then
+        namewhatchar = ' '
+      end
+      source = info.source
+      if source:sub(1, 1) == '@' then
+        if source:sub(-4, -1) == '.lua' then
+          source = source:sub(1, -5)
+        end
+        source = source:sub(-hook_sfnamelen, -1)
+      end
+      lnum = lnum or info.currentline
+    end
+
+    -- assert(-1 <= lnum and lnum <= 99999)
+    local lnum_s
+    if lnum == -1 then
+      lnum_s = 'nknwn'
+    else
+      lnum_s = ('%u'):format(lnum)
+    end
+    local msg = (  -- lua does not support %*
+      ''
+      .. msgchar
+      .. whatchar
+      .. namewhatchar
+      .. ' '
+      .. source .. (' '):rep(hook_sfnamelen - #source)
+      .. ':'
+      .. funcname .. (' '):rep(hook_fnamelen - #funcname)
+      .. ':'
+      .. ('0'):rep(hook_numlen - #lnum_s) .. lnum_s
+      .. '\n'
+    )
+    -- eq(hook_msglen, #msg)
+    sc.write(wr, msg)
+  end
+  debug.sethook(hook, trace_only_c and 'cr' or 'crl')
+end
+
+local trace_end_msg = ('E%s\n'):format((' '):rep(hook_msglen - 2))
+
+local function itp_child(wr, func)
+  init()
+  collectgarbage('stop')
+  child_sethook(wr)
+  local err, emsg = pcall(func)
+  debug.sethook()
+  collectgarbage('restart')
+  emsg = tostring(emsg)
+  sc.write(wr, trace_end_msg)
+  if not err then
+    if #emsg > 99999 then
+      emsg = emsg:sub(1, 99999)
+    end
+    sc.write(wr, ('-\n%05u\n%s'):format(#emsg, emsg))
+    deinit()
+    sc.close(wr)
+    sc.exit(1)
+  else
+    sc.write(wr, '+\n')
+    deinit()
+    sc.close(wr)
+    sc.exit(0)
+  end
+end
+
+local function check_child_err(rd)
+  local trace = {}
+  while true do
+    local traceline = sc.read(rd, hook_msglen)
+    if #traceline ~= hook_msglen then
+      if #traceline == 0 then
+        break
+      else
+        trace[#trace + 1] = 'Partial read: <' .. trace .. '>\n'
+      end
+    end
+    if traceline == trace_end_msg then
+      break
+    end
+    trace[#trace + 1] = traceline
+  end
+  local res = sc.read(rd, 2)
+  if #res ~= 2 then
+    local error
+    if #trace == 0 then
+      error = '\nTest crashed, no trace available\n'
+    else
+      error = '\nTest crashed, trace:\n' .. tracehelp
+      for i = 1, #trace do
+        error = error .. trace[i]
+      end
+    end
+    assert.just_fail(error)
+  end
+  if res == '+\n' then
+    return
+  end
+  eq('-\n', res)
+  local len_s = sc.read(rd, 5)
+  local len = tonumber(len_s)
+  neq(0, len)
+  local err = sc.read(rd, len + 1)
+  assert.just_fail(err)
+end
+
+local function itp_parent(rd, pid, allow_failure)
+  local err, emsg = pcall(check_child_err, rd)
+  sc.wait(pid)
+  sc.close(rd)
+  if not err then
+    if allow_failure then
+      io.stderr:write('Errorred out:\n' .. tostring(emsg) .. '\n')
+      os.execute([[
+        sh -c "source ci/common/test.sh
+        check_core_dumps --delete \"]] .. Paths.test_luajit_prg .. [[\""]])
+    else
+      error(emsg)
+    end
+  end
+end
+
 local function gen_itp(it)
   child_calls_mod = {}
   child_calls_mod_once = {}
@@ -518,14 +702,6 @@ local function gen_itp(it)
   preprocess_cache_mod = map(function(v) return v end, preprocess_cache_init)
   previous_defines_mod = previous_defines_init
   cdefs_mod = cdefs_init:copy()
-  local function just_fail(_)
-    return false
-  end
-  say:set('assertion.just_fail.positive', '%s')
-  say:set('assertion.just_fail.negative', '%s')
-  assert:register('assertion', 'just_fail', just_fail,
-                  'assertion.just_fail.positive',
-                  'assertion.just_fail.negative')
   local function itp(name, func, allow_failure)
     if allow_failure and os.getenv('NVIM_TEST_RUN_FAILING_TESTS') ~= '1' then
       -- FIXME Fix tests with this true
@@ -535,50 +711,13 @@ local function gen_itp(it)
       local rd, wr = sc.pipe()
       child_pid = sc.fork()
       if child_pid == 0 then
-        init()
         sc.close(rd)
-        collectgarbage('stop')
-        local err, emsg = pcall(func)
-        collectgarbage('restart')
-        emsg = tostring(emsg)
-        if not err then
-          sc.write(wr, ('-\n%05u\n%s'):format(#emsg, emsg))
-          deinit()
-          sc.close(wr)
-          sc.exit(1)
-        else
-          sc.write(wr, '+\n')
-          deinit()
-          sc.close(wr)
-          sc.exit(0)
-        end
+        itp_child(wr, func)
       else
         sc.close(wr)
-        sc.wait(child_pid)
+        local saved_child_pid = child_pid
         child_pid = nil
-        local function check()
-          local res = sc.read(rd, 2)
-          eq(2, #res)
-          if res == '+\n' then
-            return
-          end
-          eq('-\n', res)
-          local len_s = sc.read(rd, 5)
-          local len = tonumber(len_s)
-          neq(0, len)
-          local err = sc.read(rd, len + 1)
-          assert.just_fail(err)
-        end
-        local err, emsg = pcall(check)
-        sc.close(rd)
-        if not err then
-          if allow_failure then
-            io.stderr:write('Errorred out:\n' .. tostring(emsg) .. '\n')
-            os.execute([[sh -c "source ci/common/test.sh ; check_core_dumps --delete \"]] .. Paths.test_luajit_prg .. [[\""]])
-          else
-            error(emsg)
-          end
-        end
+        itp_parent(rd, saved_child_pid, allow_failure)
       end
     end)
   end
@@ -610,6 +749,7 @@ local module = {
   only_separate = only_separate,
   child_call_once = child_call_once,
   child_cleanup_once = child_cleanup_once,
+  sc = sc,
 }
 return function(after_each)
   if after_each then

test/unit/testtest_spec.lua

@@ -0,0 +1,19 @@
+local helpers = require('test.unit.helpers')(after_each)
+local assert = require('luassert')
+
+local itp = helpers.gen_itp(it)
+
+local sc = helpers.sc
+
+-- All of the below tests must fail. Check how exactly they fail.
+if os.getenv('NVIM_TEST_RUN_TESTTEST') ~= '1' then
+  return
+end
+describe('test code', function()
+  itp('does not hang when working with lengthy errors', function()
+    assert.just_fail(('x'):rep(65536))
+  end)
+  itp('shows trace after exiting abnormally', function()
+    sc.exit(0)
+  end)
+end)