test: include test path in summary (#39141)

Ref: https://github.com/neovim/neovim/pull/38486#discussion_r3088483987

runtime/doc/autocmd.txt

@@ -202,7 +202,7 @@ was last defined.  Example: >
 See |:verbose-cmd| for more information.
 
 ==============================================================================
-5. Events				*autocmd-events* *E215* *E216*
+5. Events					*autocmd-events* *E215* *E216*
 
 You can specify a comma-separated list of event names.  No white space can be
 used in this list.  The command applies to all the events in the list.

test/functional/core/server_spec.lua

@@ -287,7 +287,7 @@ describe('startup --listen', function()
     local cmd = vim.list_extend({ '--clean', '+qall!', '--headless' }, args)
     local r = run(cmd)
     eq(1, r.status)
-    matches(expected, r:output():gsub('\\n', ' '))
+    matches(expected, (r:output():gsub('\\n', ' ')))
 
     if is_os('win') then
       return -- On Windows, output without --headless is garbage.
@@ -296,7 +296,7 @@ describe('startup --listen', function()
     assert(not vim.tbl_contains(cmd, '--headless'))
     r = run(cmd)
     eq(1, r.status)
-    matches(expected, r:output():gsub('\\n', ' '))
+    matches(expected, (r:output():gsub('\\n', ' ')))
   end
 
   it('validates', function()

test/functional/core/startup_spec.lua

@@ -181,7 +181,7 @@ describe('startup', function()
 
     it('Lua-error sets Nvim exitcode', function()
       local proc = n.spawn_wait('-l', 'test/functional/fixtures/startup-fail.lua')
-      matches('E5113: .* my pearls!!', proc:output())
+      matches('E5113: .* my pearls!!', (proc:output()))
       eq(0, proc.signal)
       eq(1, proc.status)
 

test/functional/harness/harness_spec.lua

@@ -4,6 +4,8 @@ local t = require('test.testutil')
 local uv = vim.uv
 
 local eq = t.eq
+local matches = t.matches
+local not_matches = t.not_matches
 
 local root = t.paths.test_source_path
 local build_dir = t.paths.test_build_dir
@@ -131,7 +133,7 @@ end
 local function assert_harness_passes(suite_dir, extra_args)
   local code, output = run_harness(suite_dir, extra_args)
   eq(0, code)
-  eq(true, output:find('PASSED', 1, true) ~= nil)
+  matches('PASSED', output, true)
 end
 
 describe('test harness', function()
@@ -365,7 +367,7 @@ describe('test harness', function()
     })
 
     eq(1, code)
-    eq(true, not not output:find("attempt to call global 'before_each'", 1, true), output)
+    matches("attempt to call global 'before_each'", output, true)
   end)
 
   it('rejects helpers that define suites or tests', function()
@@ -387,7 +389,7 @@ describe('test harness', function()
     })
 
     eq(1, code)
-    eq(true, not not output:find("attempt to call global 'describe'", 1, true), output)
+    matches("attempt to call global 'describe'", output, true)
   end)
 
   it('deduplicates suite-end callbacks registered from the same module', function()
@@ -529,13 +531,13 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('[suite_end 1]', 1, true))
-    eq(true, not not output:find('suite_end', 1, true))
-    eq(true, not not output:find('boom from suite_end', 1, true))
-    eq(true, not not output:find('ERROR   ', 1, true))
-    eq(false, not not output:find('FAILED  ', 1, true))
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true))
-    eq(true, not not output:find('Global test environment teardown.', 1, true))
+    matches('[suite_end 1]', output, true)
+    matches('suite_end', output, true)
+    matches('boom from suite_end', output, true)
+    matches('ERROR   ', output, true)
+    not_matches('FAILED  ', output, true)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
+    matches('Global test environment teardown.', output, true)
   end)
 
   it('reports wrapped suite-end callback failures at the callback definition line', function()
@@ -573,8 +575,8 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('cbmod.lua @ 4: [suite_end 1]', 1, true), output)
-    eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output)
+    matches('cbmod.lua @ 4: [suite_end 1]', output, true)
+    not_matches('wrapper.lua @ 4:', output, true)
   end)
 
   it('reloads helper suite-end callbacks between repeats', function()
@@ -651,9 +653,9 @@ describe('test harness', function()
     })
 
     eq(0, code)
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output)
-    eq(true, not not output:find('one fast #fast', 1, true), output)
-    eq(false, not not output:find('two slow #slow', 1, true), output)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
+    matches('one fast #fast', output, true)
+    not_matches('two slow #slow', output, true)
   end)
 
   it('filters tests by suite tags across files', function()
@@ -675,9 +677,9 @@ describe('test harness', function()
     })
 
     eq(0, code)
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output)
-    eq(true, not not output:find('one #fast works', 1, true), output)
-    eq(false, not not output:find('two #slow works', 1, true), output)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
+    matches('one #fast works', output, true)
+    not_matches('two #slow works', output, true)
   end)
 
   it('filters tests by name across files', function()
@@ -699,9 +701,9 @@ describe('test harness', function()
     })
 
     eq(0, code)
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output)
-    eq(true, not not output:find('one chosen', 1, true), output)
-    eq(false, not not output:find('two skipped', 1, true), output)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
+    matches('one chosen', output, true)
+    not_matches('two skipped', output, true)
   end)
 
   it('filters tests by suite name across files', function()
@@ -724,9 +726,9 @@ describe('test harness', function()
     })
 
     eq(0, code)
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output)
-    eq(true, not not output:find('chosen suite works', 1, true), output)
-    eq(false, not not output:find('skipped suite works', 1, true), output)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
+    matches('chosen suite works', output, true)
+    not_matches('skipped suite works', output, true)
   end)
 
   it('does not keep suite-end callbacks from filtered-out files', function()
@@ -765,7 +767,7 @@ describe('test harness', function()
 
     eq(0, code)
     eq(nil, uv.fs_stat(marker))
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
   end)
 
   it('filters tests out by name across files', function()
@@ -787,9 +789,9 @@ describe('test harness', function()
     })
 
     eq(0, code)
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output)
-    eq(true, not not output:find('one chosen', 1, true), output)
-    eq(false, not not output:find('two skipped', 1, true), output)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
+    matches('one chosen', output, true)
+    not_matches('two skipped', output, true)
   end)
 
   it('filters tests out by suite name across files', function()
@@ -812,9 +814,9 @@ describe('test harness', function()
     })
 
     eq(0, code)
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output)
-    eq(true, not not output:find('chosen suite works', 1, true), output)
-    eq(false, not not output:find('skipped suite works', 1, true), output)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
+    matches('chosen suite works', output, true)
+    not_matches('skipped suite works', output, true)
   end)
 
   it('reports when filters exclude all tests', function()
@@ -831,8 +833,8 @@ describe('test harness', function()
     })
 
     eq(1, code)
-    eq(true, not not output:find('No tests matched the current selection.', 1, true), output)
-    eq(false, not not output:find('Running tests from', 1, true), output)
+    matches('No tests matched the current selection.', output, true)
+    not_matches('Running tests from', output, true)
   end)
 
   it('reports malformed filter patterns clearly before selection runs', function()
@@ -851,10 +853,10 @@ describe('test harness', function()
       local code, output = run_harness(suite_dir, { case[1] })
 
       eq(1, code)
-      eq(true, not not output:find(case[2], 1, true), output)
-      eq(false, not not output:find('test_selected', 1, true), output)
-      eq(false, not not output:find('Running tests from', 1, true), output)
-      eq(false, not not output:find('test harness failed with exit code', 1, true), output)
+      matches(case[2], output, true)
+      not_matches('test_selected', output, true)
+      not_matches('Running tests from', output, true)
+      not_matches('test harness failed with exit code', output, true)
     end
   end)
 
@@ -892,10 +894,10 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(0, code)
-    eq(true, not not output:find('Running tests from', 1, true), output)
-    eq(true, not not output:find('one works', 1, true), output)
-    eq(false, not not output:find('two is hidden behind permissions', 1, true), output)
-    eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output)
+    matches('Running tests from', output, true)
+    matches('one works', output, true)
+    not_matches('two is hidden behind permissions', output, true)
+    matches('1 test from 1 test file of ' .. suite_dir .. ' ran.', output, true)
   end)
 
   it('reports missing test paths clearly before loading suites', function()
@@ -911,10 +913,10 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir, { missing })
 
     eq(1, code)
-    eq(true, not not output:find('test path not found: ' .. missing, 1, true), output)
-    eq(false, not not output:find('collect_test_files', 1, true), output)
-    eq(false, not not output:find('Running tests from', 1, true), output)
-    eq(false, not not output:find('test harness failed with exit code', 1, true), output)
+    matches('test path not found: ' .. missing, output, true)
+    not_matches('collect_test_files', output, true)
+    not_matches('Running tests from', output, true)
+    not_matches('test harness failed with exit code', output, true)
   end)
 
   it('reports missing helpers clearly before running suites', function()
@@ -932,10 +934,10 @@ describe('test harness', function()
     })
 
     eq(1, code)
-    eq(true, not not output:find('cannot open ' .. missing, 1, true), output)
-    eq(false, not not output:find('load_helper', 1, true), output)
-    eq(false, not not output:find('Running tests from', 1, true), output)
-    eq(false, not not output:find('test harness failed with exit code', 1, true), output)
+    matches('cannot open ' .. missing, output, true)
+    not_matches('load_helper', output, true)
+    not_matches('Running tests from', output, true)
+    not_matches('test harness failed with exit code', output, true)
   end)
 
   it('reports empty value options before running suites', function()
@@ -956,10 +958,10 @@ describe('test harness', function()
       })
 
       eq(1, code)
-      eq(true, not not output:find(case[2], 1, true), output)
-      eq(false, not not output:find('open_summary_file', 1, true), output)
-      eq(false, not not output:find('Running tests from', 1, true), output)
-      eq(false, not not output:find('test harness failed with exit code', 1, true), output)
+      matches(case[2], output, true)
+      not_matches('open_summary_file', output, true)
+      not_matches('Running tests from', output, true)
+      not_matches('test harness failed with exit code', output, true)
     end
   end)
 
@@ -975,14 +977,14 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir, { '--filter', '--verbose' })
 
     eq(1, code)
-    eq(true, not not output:find('No tests matched the current selection.', 1, true), output)
-    eq(false, not not output:find('missing value for --filter', 1, true), output)
+    matches('No tests matched the current selection.', output, true)
+    not_matches('missing value for --filter', output, true)
 
     code, output = run_harness(suite_dir, { '--repeat', '-1' })
 
     eq(1, code)
-    eq(true, not not output:find('invalid value for --repeat: -1', 1, true), output)
-    eq(false, not not output:find('missing value for --repeat', 1, true), output)
+    matches('invalid value for --repeat: -1', output, true)
+    not_matches('missing value for --repeat', output, true)
   end)
 
   it('reports test-body errors as failures', function()
@@ -999,9 +1001,9 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('boom from test', 1, true))
-    eq(true, not not output:find('FAILED  ', 1, true))
-    eq(false, not not output:find('ERROR   ', 1, true))
+    matches('boom from test', output, true)
+    matches('FAILED  ', output, true)
+    not_matches('ERROR   ', output, true)
   end)
 
   it('reports test-body failures at the failing line', function()
@@ -1018,7 +1020,7 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 3: one fails', 1, true), output)
+    matches('one_spec.lua @ 3: one fails', output, true)
   end)
 
   it('ignores fake trace lines embedded in multiline error messages', function()
@@ -1035,8 +1037,8 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 3: one fake trace', 1, true), output)
-    eq(false, not not output:find('one_spec.lua @ 999: one fake trace', 1, true), output)
+    matches('one_spec.lua @ 3: one fake trace', output, true)
+    not_matches('one_spec.lua @ 999: one fake trace', output, true)
   end)
 
   it('reports wrapped assertion failures at the test definition line', function()
@@ -1053,8 +1055,8 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 2: one equal fail', 1, true), output)
-    eq(false, not not output:find(root .. '/test/assert.lua @', 1, true), output)
+    matches('one_spec.lua @ 2: one equal fail', output, true)
+    not_matches(root .. '/test/assert.lua @', output, true)
   end)
 
   it('reports local wrapper failures at the test definition line', function()
@@ -1082,8 +1084,8 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 4: one wrapper fail', 1, true), output)
-    eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output)
+    matches('one_spec.lua @ 4: one wrapper fail', output, true)
+    not_matches('wrapper.lua @ 4:', output, true)
   end)
 
   it('reports failing finally cleanup at the cleanup line', function()
@@ -1103,7 +1105,7 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 4: one finfail', 1, true), output)
+    matches('one_spec.lua @ 4: one finfail', output, true)
   end)
 
   it('rejects finally in setup hooks', function()
@@ -1122,12 +1124,8 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 2: one [setup]', 1, true), output)
-    eq(
-      true,
-      not not output:find('finally() must be called while a test body is running', 1, true),
-      output
-    )
+    matches('one_spec.lua @ 2: one [setup]', output, true)
+    matches('finally() must be called while a test body is running', output, true)
   end)
 
   it('rejects finally in teardown hooks', function()
@@ -1146,12 +1144,8 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 2: one [teardown]', 1, true), output)
-    eq(
-      true,
-      not not output:find('finally() must be called while a test body is running', 1, true),
-      output
-    )
+    matches('one_spec.lua @ 2: one [teardown]', output, true)
+    matches('finally() must be called while a test body is running', output, true)
   end)
 
   it('reports failing after_each cleanup at the cleanup line', function()
@@ -1172,7 +1166,7 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 3: one afterfail', 1, true), output)
+    matches('one_spec.lua @ 3: one afterfail', output, true)
   end)
 
   it('reports cross-file after_each wrapper failures at the hook definition line', function()
@@ -1204,9 +1198,9 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 4: one afterwrap', 1, true), output)
-    eq(false, not not output:find('one_spec.lua @ 8: one afterwrap', 1, true), output)
-    eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output)
+    matches('one_spec.lua @ 4: one afterwrap', output, true)
+    not_matches('one_spec.lua @ 8: one afterwrap', output, true)
+    not_matches('wrapper.lua @ 4:', output, true)
   end)
 
   it(
@@ -1240,9 +1234,9 @@ describe('test harness', function()
       local code, output = run_harness(suite_dir)
 
       eq(1, code)
-      eq(true, not not output:find('one_spec.lua @ 4: one afterwrap pending', 1, true), output)
-      eq(false, not not output:find('one_spec.lua @ 8: one afterwrap pending', 1, true), output)
-      eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output)
+      matches('one_spec.lua @ 4: one afterwrap pending', output, true)
+      not_matches('one_spec.lua @ 8: one afterwrap pending', output, true)
+      not_matches('wrapper.lua @ 4:', output, true)
     end
   )
 
@@ -1273,9 +1267,9 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 5: one finwrap', 1, true), output)
-    eq(false, not not output:find('one_spec.lua @ 4: one finwrap', 1, true), output)
-    eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output)
+    matches('one_spec.lua @ 5: one finwrap', output, true)
+    not_matches('one_spec.lua @ 4: one finwrap', output, true)
+    not_matches('wrapper.lua @ 4:', output, true)
   end)
 
   it('does not count synthetic hook failures as executed tests', function()
@@ -1294,8 +1288,8 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('0 tests from 1 test file ran.', 1, true))
-    eq(true, not not output:find('1 ERROR', 1, true))
+    matches('0 tests from 1 test file of ' .. suite_dir .. ' ran.', output, true)
+    matches('1 ERROR', output, true)
   end)
 
   it('reports setup failures at the hook definition site', function()
@@ -1314,7 +1308,7 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, not not output:find('one_spec.lua @ 2: trace [setup]', 1, true), output)
+    matches('one_spec.lua @ 2: trace [setup]', output, true)
   end)
 
   it('does not keep suite-end callbacks from load-error files', function()
@@ -1340,7 +1334,7 @@ describe('test harness', function()
 
     eq(1, code)
     eq(nil, uv.fs_stat(marker))
-    eq(true, not not output:find('boom during load', 1, true), output)
+    matches('boom during load', output, true)
   end)
 
   it('reports load failures at the failing line', function()
@@ -1355,7 +1349,7 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, output:match('one_spec%.lua @ 3: .-one_spec%.lua %[load%]') ~= nil, output)
+    matches('one_spec%.lua @ 3: .-one_spec%.lua %[load%]', output)
   end)
 
   it('reports required-module syntax errors at the real module line', function()
@@ -1377,6 +1371,6 @@ describe('test harness', function()
     local code, output = run_harness(suite_dir)
 
     eq(1, code)
-    eq(true, output:match('broken%.lua @ 3: .-one_spec%.lua %[load%]') ~= nil, output)
+    matches('broken%.lua @ 3: .-one_spec%.lua %[load%]', output)
   end)
 end)

test/harness.lua

@@ -1530,6 +1530,7 @@ end
 --- @return test.harness.IterationResult?, string?
 local function run_iteration(Reporter, opts, files, pre_helper_baseline, repeat_index)
   local reporter = Reporter.new({
+    paths = opts.paths,
     verbose = opts.verbose,
     summary_file = opts.summary_file,
   })

test/reporter.lua

@@ -14,6 +14,7 @@
 --- @field duration? number
 
 --- @class test.reporter.Options
+--- @field paths string[]
 --- @field verbose boolean
 --- @field summary_file string
 
@@ -436,15 +437,20 @@ function M:suite_end(duration, run_summary, failure_output)
   io.write(('%s Global test environment teardown.\n'):format(self:sect('--------')))
   io.flush()
 
+  local fpath = function(s)
+    return self:fpath(s)
+  end
+
   local summary_file = open_summary_file(self.opts.summary_file)
   summary_file:write('\n')
   summary_file:write(
-    ('%s %s %s from %s test %s ran. %s\n'):format(
+    ('%s %s %s from %s test %s of %s ran. %s\n'):format(
       self:sect('========'),
       self:nmbr(run_summary.test_count),
       tests,
       self:nmbr(run_summary.file_count),
       files,
+      vim.iter(self.opts.paths):map(fpath):join(';'),
       self:time(('(%.2f ms total)'):format(duration * 1000))
     )
   )

test/testutil.lua

@@ -126,13 +126,27 @@ end
 
 --- @param pat string
 --- @param actual string
+--- @param plain boolean? (default: false)
 --- @return boolean
-function M.matches(pat, actual)
+function M.matches(pat, actual, plain)
   assert(pat and pat ~= '', 'pat must be a non-empty string')
-  if nil ~= string.match(actual, pat) then
+  assert(plain == nil or type(plain) == 'boolean', 'plain must be nil or boolean')
+  if nil ~= string.find(actual, pat, 1, plain) then
     return true
   end
-  error(string.format('Pattern does not match.\nPattern:\n%s\nActual:\n%s', pat, actual))
+  error(('Pattern does not match.\nPattern:\n%s\nActual:\n%s'):format(pat, actual))
+end
+
+--- @param pat string
+--- @param actual string
+--- @param plain boolean? (default: false)
+--- @return boolean
+function M.not_matches(pat, actual, plain)
+  assert(pat and pat ~= '', 'pat must be a non-empty string')
+  if nil == string.find(actual, pat, 1, plain) then
+    return true
+  end
+  error(('Pattern does match.\nPattern:\n%s\nActual:\n%s'):format(pat, actual))
 end
 
 --- Asserts that `pat` matches (or *not* if inverse=true) any text in the tail of `logfile`.