test: support running functionaltests in parallel by directory (#37918)

Define a CMake target for every subdirectory of test/functional that contains functional tests, and a functionaltest_parallel target that depends on all those targets, allowing multiple test runners to run in parallel. On CI, use at most 2 parallel test runners, as using more may increase system load and make tests unstable.
2026-08-27 09:31:47 +00:00 · 2026-02-18 15:56:50 +08:00
parent 9492df027d
commit 496eca22b3
13 changed files with 206 additions and 102 deletions
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -2,13 +2,14 @@ add_subdirectory(functional/fixtures)  # compile test programs

 get_directory_property(TEST_INCLUDE_DIRS DIRECTORY ${PROJECT_SOURCE_DIR}/src/nvim DEFINITION TEST_INCLUDE_DIRS)

+set(TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(TEST_OPTIONS
      -D BUILD_DIR=${CMAKE_BINARY_DIR}
      -D CIRRUS_CI=$ENV{CIRRUS_CI}
      -D CI_BUILD=${CI_BUILD}
      -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
      -D NVIM_PRG=$<TARGET_FILE:nvim_bin>
-      -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
+      -D TEST_DIR=${TEST_DIR}
      -D WORKING_DIR=${PROJECT_SOURCE_DIR})

 check_lua_module(${LUA_PRG} "ffi" LUA_HAS_FFI)
@@ -28,6 +29,15 @@ configure_file(
  ${CMAKE_SOURCE_DIR}/test/cmakeconfig/paths.lua.in
  ${CMAKE_BINARY_DIR}/test/cmakeconfig/paths.lua)

+add_custom_target(benchmark
+  COMMAND ${CMAKE_COMMAND}
+    -D TEST_TYPE=benchmark
+    ${TEST_OPTIONS}
+    -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake
+  DEPENDS tty-test
+  USES_TERMINAL)
+add_dependencies(benchmark lua_dev_deps nvim)
+
 add_custom_target(functionaltest
  COMMAND ${CMAKE_COMMAND}
    -D TEST_TYPE=functional
@@ -37,11 +47,30 @@ add_custom_target(functionaltest
  USES_TERMINAL)
 add_dependencies(functionaltest lua_dev_deps nvim)

-add_custom_target(benchmark
-  COMMAND ${CMAKE_COMMAND}
-    -D TEST_TYPE=benchmark
-    ${TEST_OPTIONS}
-    -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake
-  DEPENDS tty-test
-  USES_TERMINAL)
-add_dependencies(benchmark lua_dev_deps nvim)
+# Create multiple targets for groups of functional tests to enable parallel testing.
+set(group_targets "")
+set(summary_files "")
+file(GLOB_RECURSE test_files RELATIVE "${TEST_DIR}/functional" "${TEST_DIR}/functional/*_spec*")
+foreach(test_file ${test_files})
+  # Get test group: "test/functional/foo/bar_spec.lua" => "foo".
+  string(REGEX REPLACE "/.*" "" test_group ${test_file})
+  set(group_target "functionaltest_${test_group}")
+  string(REGEX REPLACE "[^A-Za-z0-9_]" "_" group_target ${group_target})
+  list(FIND group_targets ${group_target} group_idx)
+  if(${group_idx} EQUAL -1)
+    # Create new target for test group.
+    set(summary_file "${CMAKE_BINARY_DIR}/${group_target}.summary")
+    add_custom_target(${group_target}
+      COMMAND ${CMAKE_COMMAND}
+        -D TEST_TYPE=functional -D TEST_PARALLEL_GROUP=${test_group} -D TEST_SUMMARY_FILE=${summary_file}
+        ${TEST_OPTIONS}
+        -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake
+      DEPENDS printenv-test printargs-test shell-test pwsh-test streams-test tty-test)
+    add_dependencies(${group_target} lua_dev_deps nvim)
+    list(APPEND group_targets ${group_target})
+    list(APPEND summary_files ${summary_file})
+  endif()
+endforeach()
+
+add_custom_target(functionaltest_parallel DEPENDS ${group_targets})
+add_custom_target(functionaltest_summary COMMAND ${CMAKE_COMMAND} -E cat ${summary_files} USES_TERMINAL)
--- a/test/busted/outputHandlers/nvim.lua
+++ b/test/busted/outputHandlers/nvim.lua
@@ -25,6 +25,8 @@ end
 return function(options)
  local busted = require 'busted'
  local handler = require 'busted.outputHandlers.base'()
+  local args = options.arguments
+  args = vim.json.decode(#args > 0 and table.concat(args, ',') or '{}')

  local c = {
    succ = function(s)
@@ -254,13 +256,25 @@ return function(options)
    local elapsedTime_ms = getElapsedTime(suite)
    local tests = (testCount == 1 and 'test' or 'tests')
    local files = (fileCount == 1 and 'file' or 'files')
-    io.write(globalTeardown)
-    io.write(suiteEndString:format(testCount, tests, fileCount, files, elapsedTime_ms))
-    io.write(getSummaryString())
-    if failureCount > 0 or errorCount > 0 then
-      io.write(t_global.read_nvim_log(nil, true))
+    if type(args.test_path) == 'string' then
+      files = files .. ' of ' .. args.test_path
    end
+    local sf = type(args.summary_file) == 'string'
+        and args.summary_file ~= '-'
+        and io.open(args.summary_file, 'w')
+      or io.stdout
+    io.write(globalTeardown)
    io.flush()
+    sf:write('\n')
+    sf:write(suiteEndString:format(testCount, tests, fileCount, files, elapsedTime_ms))
+    sf:write(getSummaryString())
+    if failureCount > 0 or errorCount > 0 then
+      sf:write(t_global.read_nvim_log(nil, true))
+    end
+    sf:flush()
+    if sf ~= io.stdout then
+      sf:close()
+    end

    return nil, true
  end
--- a/test/functional/api/vim_spec.lua
+++ b/test/functional/api/vim_spec.lua
@@ -734,23 +734,24 @@ describe('API', function()

  describe('nvim_set_current_dir', function()
    local start_dir
+    local test_dir = 'Xtest_set_current_dir'

    before_each(function()
-      fn.mkdir('Xtestdir')
+      fn.mkdir(test_dir)
      start_dir = fn.getcwd()
    end)

    after_each(function()
-      n.rmdir('Xtestdir')
+      n.rmdir(test_dir)
    end)

    it('works', function()
-      api.nvim_set_current_dir('Xtestdir')
-      eq(start_dir .. n.get_pathsep() .. 'Xtestdir', fn.getcwd())
+      api.nvim_set_current_dir(test_dir)
+      eq(start_dir .. n.get_pathsep() .. test_dir, fn.getcwd())
    end)

    it('sets previous directory', function()
-      api.nvim_set_current_dir('Xtestdir')
+      api.nvim_set_current_dir(test_dir)
      command('cd -')
      eq(start_dir, fn.getcwd())
    end)
@@ -1670,8 +1671,9 @@ describe('API', function()

      -- Check if autoload works properly
      local pathsep = n.get_pathsep()
-      local xconfig = 'Xhome' .. pathsep .. 'Xconfig'
-      local xdata = 'Xhome' .. pathsep .. 'Xdata'
+      local xhome = 'Xhome_api'
+      local xconfig = xhome .. pathsep .. 'Xconfig'
+      local xdata = xhome .. pathsep .. 'Xdata'
      local autoload_folder = table.concat({ xconfig, 'nvim', 'autoload' }, pathsep)
      local autoload_file = table.concat({ autoload_folder, 'testload.vim' }, pathsep)
      mkdir_p(autoload_folder)
@@ -1679,7 +1681,7 @@ describe('API', function()

      clear { args_rm = { '-u' }, env = { XDG_CONFIG_HOME = xconfig, XDG_DATA_HOME = xdata } }
      eq(2, api.nvim_get_var('testload#value'))
-      rmdir('Xhome')
+      rmdir(xhome)
    end)

    it('nvim_get_vvar, nvim_set_vvar', function()
@@ -2966,16 +2968,18 @@ describe('API', function()
  end)

  describe('nvim_list_runtime_paths', function()
+    local test_dir = 'Xtest_list_runtime_paths'
+
    setup(function()
      local pathsep = n.get_pathsep()
-      mkdir_p('Xtest' .. pathsep .. 'a')
-      mkdir_p('Xtest' .. pathsep .. 'b')
+      mkdir_p(test_dir .. pathsep .. 'a')
+      mkdir_p(test_dir .. pathsep .. 'b')
    end)
    teardown(function()
-      rmdir 'Xtest'
+      rmdir(test_dir)
    end)
    before_each(function()
-      api.nvim_set_current_dir 'Xtest'
+      api.nvim_set_current_dir(test_dir)
    end)

    it('returns nothing with empty &runtimepath', function()
--- a/test/functional/ex_cmds/trust_spec.lua
+++ b/test/functional/ex_cmds/trust_spec.lua
@@ -11,57 +11,63 @@ local is_os = t.is_os
 local fn = n.fn

 describe(':trust', function()
-  local xstate = 'Xstate'
+  local xstate = 'Xstate_ex_trust'
+  local test_file = 'Xtest_functional_ex_cmds_trust'

  before_each(function()
    n.mkdir_p(xstate .. pathsep .. (is_os('win') and 'nvim-data' or 'nvim'))
-    t.write_file('test_file', 'test')
+    t.write_file(test_file, 'test')
    clear { env = { XDG_STATE_HOME = xstate } }
  end)

  after_each(function()
-    os.remove('test_file')
+    os.remove(test_file)
    n.rmdir(xstate)
  end)

+  --- @param s string
+  local function fmt(s)
+    return s:format(test_file)
+  end
+
  it('is not executed when inside false condition', function()
-    command('edit test_file')
+    command(fmt('edit %s'))
    eq('', exec_capture('if 0 | trust | endif'))
    eq(nil, vim.uv.fs_stat(fn.stdpath('state') .. pathsep .. 'trust'))
  end)

  it('trust then deny then remove a file using current buffer', function()
    local cwd = fn.getcwd()
-    local hash = fn.sha256(t.read_file('test_file'))
+    local hash = fn.sha256(assert(t.read_file(test_file)))

-    command('edit test_file')
-    matches('^Allowed in trust database%: ".*test_file"$', exec_capture('trust'))
+    command(fmt('edit %s'))
+    matches(fmt('^Allowed in trust database%%: ".*%s"$'), exec_capture('trust'))
    local trust = t.read_file(fn.stdpath('state') .. pathsep .. 'trust')
-    eq(string.format('%s %s', hash, cwd .. pathsep .. 'test_file'), vim.trim(trust))
+    eq(string.format('%s %s', hash, cwd .. pathsep .. test_file), vim.trim(trust))

-    matches('^Denied in trust database%: ".*test_file"$', exec_capture('trust ++deny'))
+    matches(fmt('^Denied in trust database%%: ".*%s"$'), exec_capture('trust ++deny'))
    trust = t.read_file(fn.stdpath('state') .. pathsep .. 'trust')
-    eq(string.format('! %s', cwd .. pathsep .. 'test_file'), vim.trim(trust))
+    eq(string.format('! %s', cwd .. pathsep .. test_file), vim.trim(trust))

-    matches('^Removed from trust database%: ".*test_file"$', exec_capture('trust ++remove'))
+    matches(fmt('^Removed from trust database%%: ".*%s"$'), exec_capture('trust ++remove'))
    trust = t.read_file(fn.stdpath('state') .. pathsep .. 'trust')
    eq(string.format(''), vim.trim(trust))
  end)

  it('deny then trust then remove a file using current buffer', function()
    local cwd = fn.getcwd()
-    local hash = fn.sha256(t.read_file('test_file'))
+    local hash = fn.sha256(assert(t.read_file(test_file)))

-    command('edit test_file')
-    matches('^Denied in trust database%: ".*test_file"$', exec_capture('trust ++deny'))
+    command(fmt('edit %s'))
+    matches(fmt('^Denied in trust database%%: ".*%s"$'), exec_capture('trust ++deny'))
    local trust = t.read_file(fn.stdpath('state') .. pathsep .. 'trust')
-    eq(string.format('! %s', cwd .. pathsep .. 'test_file'), vim.trim(trust))
+    eq(string.format('! %s', cwd .. pathsep .. test_file), vim.trim(trust))

-    matches('^Allowed in trust database%: ".*test_file"$', exec_capture('trust'))
+    matches(fmt('^Allowed in trust database%%: ".*%s"$'), exec_capture('trust'))
    trust = t.read_file(fn.stdpath('state') .. pathsep .. 'trust')
-    eq(string.format('%s %s', hash, cwd .. pathsep .. 'test_file'), vim.trim(trust))
+    eq(string.format('%s %s', hash, cwd .. pathsep .. test_file), vim.trim(trust))

-    matches('^Removed from trust database%: ".*test_file"$', exec_capture('trust ++remove'))
+    matches(fmt('^Removed from trust database%%: ".*%s"$'), exec_capture('trust ++remove'))
    trust = t.read_file(fn.stdpath('state') .. pathsep .. 'trust')
    eq(string.format(''), vim.trim(trust))
  end)
@@ -69,14 +75,11 @@ describe(':trust', function()
  it('deny then remove a file using file path', function()
    local cwd = fn.getcwd()

-    matches('^Denied in trust database%: ".*test_file"$', exec_capture('trust ++deny test_file'))
+    matches(fmt('^Denied in trust database%%: ".*%s"$'), exec_capture(fmt('trust ++deny %s')))
    local trust = t.read_file(fn.stdpath('state') .. pathsep .. 'trust')
-    eq(string.format('! %s', cwd .. pathsep .. 'test_file'), vim.trim(trust))
+    eq(string.format('! %s', cwd .. pathsep .. test_file), vim.trim(trust))

-    matches(
-      '^Removed from trust database%: ".*test_file"$',
-      exec_capture('trust ++remove test_file')
-    )
+    matches(fmt('^Removed from trust database%%: ".*%s"$'), exec_capture(fmt('trust ++remove %s')))
    trust = t.read_file(fn.stdpath('state') .. pathsep .. 'trust')
    eq(string.format(''), vim.trim(trust))
  end)
--- a/test/functional/legacy/normal_spec.lua
+++ b/test/functional/legacy/normal_spec.lua
@@ -143,7 +143,7 @@ describe('normal', function()
    fn.mkdir(locale_dir, 'p')
    fn.filecopy(build_dir .. '/src/nvim/po/tr.mo', locale_dir .. '/nvim.mo')
    finally(function()
-      n.rmdir(build_dir .. '/share')
+      n.rmdir(vim.fs.dirname(locale_dir))
    end)

    clear({ env = { LANG = 'tr_TR.UTF-8' } })
--- a/test/functional/lua/option_and_var_spec.lua
+++ b/test/functional/lua/option_and_var_spec.lua
@@ -128,8 +128,9 @@ describe('lua stdlib', function()

      -- Check if autoload works properly
      local pathsep = n.get_pathsep()
-      local xconfig = 'Xhome' .. pathsep .. 'Xconfig'
-      local xdata = 'Xhome' .. pathsep .. 'Xdata'
+      local xhome = 'Xhome_lua'
+      local xconfig = xhome .. pathsep .. 'Xconfig'
+      local xdata = xhome .. pathsep .. 'Xdata'
      local autoload_folder = table.concat({ xconfig, 'nvim', 'autoload' }, pathsep)
      local autoload_file = table.concat({ autoload_folder, 'testload.vim' }, pathsep)
      mkdir_p(autoload_folder)
@@ -138,7 +139,7 @@ describe('lua stdlib', function()
      clear { args_rm = { '-u' }, env = { XDG_CONFIG_HOME = xconfig, XDG_DATA_HOME = xdata } }

      eq(2, exec_lua("return vim.g['testload#value']"))
-      rmdir('Xhome')
+      rmdir(xhome)
    end)

    it('vim.b', function()
--- a/test/functional/lua/secure_spec.lua
+++ b/test/functional/lua/secure_spec.lua
@@ -19,7 +19,7 @@ local read_file = t.read_file

 describe('vim.secure', function()
  describe('read()', function()
-    local xstate = 'Xstate'
+    local xstate = 'Xstate_lua_secure'
    local screen ---@type test.functional.ui.screen

    before_each(function()
@@ -272,7 +272,9 @@ describe('vim.secure', function()
  end)

  describe('trust()', function()
-    local xstate = 'Xstate'
+    local xstate = 'Xstate_lua_secure'
+    local test_file = 'Xtest_functional_lua_secure'
+    local test_dir = 'Xtest_functional_lua_secure_dir'

    setup(function()
      clear { env = { XDG_STATE_HOME = xstate } }
@@ -280,20 +282,20 @@ describe('vim.secure', function()

    before_each(function()
      n.mkdir_p(xstate .. pathsep .. (is_os('win') and 'nvim-data' or 'nvim'))
-      t.write_file('test_file', 'test')
-      t.mkdir('test_dir')
+      t.write_file(test_file, 'test')
+      t.mkdir(test_dir)
    end)

    after_each(function()
-      os.remove('test_file')
-      n.rmdir('test_dir')
+      os.remove(test_file)
+      n.rmdir(test_dir)
      n.rmdir(xstate)
    end)

    it('returns error when passing both path and bufnr', function()
      matches(
        '"path" and "bufnr" are mutually exclusive',
-        pcall_err(exec_lua, [[vim.secure.trust({action='deny', bufnr=0, path='test_file'})]])
+        pcall_err(exec_lua, [[vim.secure.trust({action='deny', bufnr=0, path=...})]], test_file)
      )
    end)

@@ -306,10 +308,10 @@ describe('vim.secure', function()

    it('trust then deny then remove a file using bufnr', function()
      local cwd = fn.getcwd()
-      local hash = fn.sha256(assert(read_file('test_file')))
-      local full_path = cwd .. pathsep .. 'test_file'
+      local hash = fn.sha256(assert(read_file(test_file)))
+      local full_path = cwd .. pathsep .. test_file

-      command('edit test_file')
+      command('edit ' .. test_file)
      eq({ true, full_path }, exec_lua([[return {vim.secure.trust({action='allow', bufnr=0})}]]))
      local trust = assert(read_file(stdpath('state') .. pathsep .. 'trust'))
      eq(string.format('%s %s', hash, full_path), vim.trim(trust))
@@ -325,10 +327,10 @@ describe('vim.secure', function()

    it('deny then trust then remove a file using bufnr', function()
      local cwd = fn.getcwd()
-      local hash = fn.sha256(assert(read_file('test_file')))
-      local full_path = cwd .. pathsep .. 'test_file'
+      local hash = fn.sha256(assert(read_file(test_file)))
+      local full_path = cwd .. pathsep .. test_file

-      command('edit test_file')
+      command('edit ' .. test_file)
      eq({ true, full_path }, exec_lua([[return {vim.secure.trust({action='deny', bufnr=0})}]]))
      local trust = assert(read_file(stdpath('state') .. pathsep .. 'trust'))
      eq(string.format('! %s', full_path), vim.trim(trust))
@@ -344,24 +346,24 @@ describe('vim.secure', function()

    it('trust using bufnr then deny then remove a file using path', function()
      local cwd = fn.getcwd()
-      local hash = fn.sha256(assert(read_file('test_file')))
-      local full_path = cwd .. pathsep .. 'test_file'
+      local hash = fn.sha256(assert(read_file(test_file)))
+      local full_path = cwd .. pathsep .. test_file

-      command('edit test_file')
+      command('edit ' .. test_file)
      eq({ true, full_path }, exec_lua([[return {vim.secure.trust({action='allow', bufnr=0})}]]))
      local trust = assert(read_file(stdpath('state') .. pathsep .. 'trust'))
      eq(string.format('%s %s', hash, full_path), vim.trim(trust))

      eq(
        { true, full_path },
-        exec_lua([[return {vim.secure.trust({action='deny', path='test_file'})}]])
+        exec_lua([[return {vim.secure.trust({action='deny', path=...})}]], test_file)
      )
      trust = assert(read_file(stdpath('state') .. pathsep .. 'trust'))
      eq(string.format('! %s', full_path), vim.trim(trust))

      eq(
        { true, full_path },
-        exec_lua([[return {vim.secure.trust({action='remove', path='test_file'})}]])
+        exec_lua([[return {vim.secure.trust({action='remove', path=...})}]], test_file)
      )
      trust = assert(read_file(stdpath('state') .. pathsep .. 'trust'))
      eq('', vim.trim(trust))
@@ -369,13 +371,13 @@ describe('vim.secure', function()

    it('deny then trust then remove a file using bufnr', function()
      local cwd = fn.getcwd()
-      local hash = fn.sha256(assert(read_file('test_file')))
-      local full_path = cwd .. pathsep .. 'test_file'
+      local hash = fn.sha256(assert(read_file(test_file)))
+      local full_path = cwd .. pathsep .. test_file

-      command('edit test_file')
+      command('edit ' .. test_file)
      eq(
        { true, full_path },
-        exec_lua([[return {vim.secure.trust({action='deny', path='test_file'})}]])
+        exec_lua([[return {vim.secure.trust({action='deny', path=...})}]], test_file)
      )
      local trust = assert(read_file(stdpath('state') .. pathsep .. 'trust'))
      eq(string.format('! %s', full_path), vim.trim(trust))
@@ -386,7 +388,7 @@ describe('vim.secure', function()

      eq(
        { true, full_path },
-        exec_lua([[return {vim.secure.trust({action='remove', path='test_file'})}]])
+        exec_lua([[return {vim.secure.trust({action='remove', path=...})}]], test_file)
      )
      trust = assert(read_file(stdpath('state') .. pathsep .. 'trust'))
      eq('', vim.trim(trust))
@@ -402,8 +404,8 @@ describe('vim.secure', function()

    it('trust then deny then remove a directory using bufnr', function()
      local cwd = fn.getcwd()
-      local full_path = cwd .. pathsep .. 'test_dir'
-      command('edit test_dir')
+      local full_path = cwd .. pathsep .. test_dir
+      command('edit ' .. test_dir)

      eq({ true, full_path }, exec_lua([[return {vim.secure.trust({action='allow', bufnr=0})}]]))
      local trust = assert(read_file(stdpath('state') .. pathsep .. 'trust'))
--- a/test/functional/plugin/spellfile_spec.lua
+++ b/test/functional/plugin/spellfile_spec.lua
@@ -6,8 +6,8 @@ local neq = t.neq
 local exec_lua = n.exec_lua

 describe('nvim.spellfile', function()
-  local data_root = 'Xtest_data'
-  local rtp_dir = 'Xtest_rtp'
+  local data_root = 'Xtest_data_spellfile'
+  local rtp_dir = 'Xtest_rtp_spellfile'

  before_each(function()
    n.clear()
--- a/test/functional/ui/messages_spec.lua
+++ b/test/functional/ui/messages_spec.lua
@@ -2447,7 +2447,7 @@ describe('ui/msg_puts_printf', function()
    fn.mkdir(locale_dir, 'p')
    fn.filecopy(build_dir .. '/src/nvim/po/ja.mo', locale_dir .. '/nvim.mo')
    finally(function()
-      n.rmdir(build_dir .. '/share')
+      n.rmdir(vim.fs.dirname(locale_dir))
    end)

    cmd = cmd .. '"' .. nvim_prog .. '" -u NONE -i NONE -Es -V1'
--- a/test/functional/vimscript/executable_spec.lua
+++ b/test/functional/vimscript/executable_spec.lua
@@ -33,9 +33,15 @@ describe('executable()', function()
    end)

    it('stdpath respects shellslash', function()
-      eq([[build\Xtest_xdg\share\nvim-data]], call('fnamemodify', call('stdpath', 'data'), ':.'))
+      t.matches(
+        [[build\Xtest_xdg[%w_]*\share\nvim%-data]],
+        call('fnamemodify', call('stdpath', 'data'), ':.')
+      )
      command('set shellslash')
-      eq('build/Xtest_xdg/share/nvim-data', call('fnamemodify', call('stdpath', 'data'), ':.'))
+      t.matches(
+        'build/Xtest_xdg[%w_]*/share/nvim%-data',
+        call('fnamemodify', call('stdpath', 'data'), ':.')
+      )
    end)
  end