build: move luarocks and rocks installation to main build

This will ensure luacheck and busted are only installed when they're
actually needed. This cuts total build time by over 50%.

Closes https://github.com/neovim/neovim/issues/22797.

CMakeLists.txt

@@ -35,6 +35,9 @@ include(InstallHelpers)
 include(PreventInTreeBuilds)
 include(Util)
 
+set_directory_properties(PROPERTIES
+  EP_PREFIX "${DEPS_BUILD_DIR}")
+
 set(TOUCHES_DIR ${PROJECT_BINARY_DIR}/touches)
 
 find_program(CCACHE_PRG ccache)
@@ -229,18 +232,18 @@ endif()
 #
 # Lint
 #
-find_program(LUACHECK_PRG luacheck)
 find_program(SHELLCHECK_PRG shellcheck)
 find_program(STYLUA_PRG stylua)
 
 add_glob_target(
   REQUIRED
   TARGET lintlua-luacheck
-  COMMAND ${LUACHECK_PRG}
+  COMMAND ${DEPS_BIN_DIR}/luacheck
   FLAGS -q
   GLOB_DIRS runtime/ scripts/ src/ test/
   GLOB_PAT *.lua
   TOUCH_STRATEGY SINGLE)
+add_dependencies(lintlua-luacheck luacheck)
 
 add_glob_target(
   TARGET lintlua-stylua
@@ -313,7 +316,6 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
 endif()
 
 ExternalProject_Add(uncrustify
-  PREFIX ${DEPS_BUILD_DIR}
   URL https://github.com/uncrustify/uncrustify/archive/uncrustify-0.77.1.tar.gz
   URL_HASH SHA256=414bbc9f7860eb18a53074f9af14ed04638a633b2216a73f2629291300d37c1b
   DOWNLOAD_NO_PROGRESS TRUE
@@ -321,3 +323,5 @@ ExternalProject_Add(uncrustify
   CMAKE_ARGS ${DEPS_CMAKE_ARGS}
   CMAKE_CACHE_ARGS ${DEPS_CMAKE_CACHE_ARGS}
   EXCLUDE_FROM_ALL TRUE)
+
+include(BuildLuarocks)

cmake.deps/CMakeLists.txt

@@ -38,7 +38,6 @@ option(USE_BUNDLED_LIBVTERM "Use the bundled libvterm." ${USE_BUNDLED})
 option(USE_BUNDLED_LIBUV "Use the bundled libuv." ${USE_BUNDLED})
 option(USE_BUNDLED_MSGPACK "Use the bundled msgpack." ${USE_BUNDLED})
 option(USE_BUNDLED_LUAJIT "Use the bundled version of luajit." ${USE_BUNDLED})
-option(USE_BUNDLED_LUAROCKS "Use the bundled version of luarocks." ${USE_BUNDLED})
 option(USE_BUNDLED_LUV "Use the bundled version of luv." ${USE_BUNDLED})
 option(USE_BUNDLED_LPEG "Use the bundled lpeg." ${USE_BUNDLED})
 #XXX(tarruda): Lua is only used for debugging the functional test client, don't
@@ -66,45 +65,6 @@ if(NOT Git_FOUND)
   message(FATAL_ERROR "Git is required to apply patches.")
 endif()
 
-if(UNIX)
-  find_program(MAKE_PRG NAMES gmake make)
-  if(NOT MAKE_PRG)
-    message(FATAL_ERROR "GNU Make is required to build the dependencies.")
-  else()
-    message(STATUS "Found GNU Make at ${MAKE_PRG}")
-  endif()
-endif()
-
-# When using make, use the $(MAKE) variable to avoid warning about the job
-# server.
-if(CMAKE_GENERATOR MATCHES "Makefiles")
-  set(MAKE_PRG "$(MAKE)")
-endif()
-
-if(MINGW AND CMAKE_GENERATOR MATCHES "Ninja")
-  find_program(MAKE_PRG NAMES mingw32-make)
-  if(NOT MAKE_PRG)
-    message(FATAL_ERROR "GNU Make for mingw32 is required to build the dependencies.")
-  else()
-    message(STATUS "Found GNU Make for mingw32: ${MAKE_PRG}")
-  endif()
-endif()
-
-set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}")
-
-if(CMAKE_OSX_SYSROOT)
-  set(DEPS_C_COMPILER "${DEPS_C_COMPILER} -isysroot${CMAKE_OSX_SYSROOT}")
-endif()
-
-if(CMAKE_OSX_ARCHITECTURES)
-  # The LuaJIT build does not like being passed multiple `-arch` flags
-  # so we handle a universal build the old-fashioned way.
-  set(LUAJIT_C_COMPILER "${DEPS_C_COMPILER}")
-  foreach(ARCH IN LISTS CMAKE_OSX_ARCHITECTURES)
-    set(DEPS_C_COMPILER "${DEPS_C_COMPILER} -arch ${ARCH}")
-  endforeach()
-endif()
-
 # If the macOS deployment target is not set manually (via $MACOSX_DEPLOYMENT_TARGET),
 # fall back to local system version. Needs to be done here and in top-level CMakeLists.txt.
 if(APPLE)
@@ -163,10 +123,6 @@ if(USE_BUNDLED_LUA)
   include(BuildLua)
 endif()
 
-if(USE_BUNDLED_LUAROCKS)
-  include(BuildLuarocks)
-endif()
-
 if(USE_BUNDLED_LUV)
   include(BuildLuv)
 endif()

cmake.deps/cmake/BuildLua.cmake

@@ -51,12 +51,3 @@ ExternalProject_Add(lua
   BUILD_IN_SOURCE 1
   BUILD_COMMAND ${MAKE_PRG} ${LUA_INSTALL_TOP_ARG} ${LUA_TARGET}
   INSTALL_COMMAND ${MAKE_PRG} ${LUA_INSTALL_TOP_ARG} install)
-
-set(BUSTED ${DEPS_BIN_DIR}/busted)
-set(BUSTED_LUA ${BUSTED}-lua)
-
-add_custom_command(OUTPUT ${BUSTED_LUA}
-  COMMAND sed -e 's/^exec/exec $$LUA_DEBUGGER/' -e 's/jit//g' < ${BUSTED} > ${BUSTED_LUA} && chmod +x ${BUSTED_LUA}
-  DEPENDS lua busted ${BUSTED})
-add_custom_target(busted-lua ALL
-  DEPENDS ${DEPS_BIN_DIR}/busted-lua)

cmake.deps/cmake/BuildLuajit.cmake

@@ -66,6 +66,11 @@ if((UNIX AND NOT APPLE) OR (APPLE AND NOT CMAKE_OSX_ARCHITECTURES))
 
 elseif(CMAKE_OSX_ARCHITECTURES AND APPLE)
 
+  set(LUAJIT_C_COMPILER "${CMAKE_C_COMPILER}")
+  if(CMAKE_OSX_SYSROOT)
+    set(LUAJIT_C_COMPILER "${LUAJIT_C_COMPILER} -isysroot${CMAKE_OSX_SYSROOT}")
+  endif()
+
   # Passing multiple `-arch` flags to the LuaJIT build will cause it to fail.
   # To get a working universal build, we build each requested architecture slice
   # individually then `lipo` them all up.

cmake.deps/deps.txt

@@ -10,9 +10,6 @@ LUAJIT_SHA256 a9bcd9e646e2b188e1d7e3fb594e04c61dda3b332dfd0378d41be19c1eae9d09
 LUA_URL https://www.lua.org/ftp/lua-5.1.5.tar.gz
 LUA_SHA256 2640fc56a795f29d28ef15e13c34a47e223960b0240e8cb0a82d9b0738695333
 
-LUAROCKS_URL https://github.com/luarocks/luarocks/archive/v3.9.2.tar.gz
-LUAROCKS_SHA256 a0b36cd68586cd79966d0106bb2e5a4f5523327867995fd66bee4237062b3e3b
-
 UNIBILIUM_URL https://github.com/neovim/unibilium/archive/d72c3598e7ac5d1ebf86ee268b8b4ed95c0fa628.tar.gz
 UNIBILIUM_SHA256 9c4747c862ab5e3076dcf8fa8f0ea7a6b50f20ec5905618b9536655596797487
 

cmake/BuildLuarocks.cmake

@@ -13,66 +13,41 @@ if(NOT MSVC)
   set(LUAROCKS_BUILDARGS CC=${DEPS_C_COMPILER} LD=${DEPS_C_COMPILER})
 endif()
 
-# Lua version, used with rocks directories.
-# Defaults to 5.1 for bundled LuaJIT/Lua.
-set(LUA_VERSION "5.1")
-
 if(UNIX)
-
+  if(PREFER_LUA)
-  if(USE_BUNDLED_LUAJIT)
+    find_package(Lua 5.1 EXACT REQUIRED)
+    get_filename_component(LUA_ROOT ${LUA_INCLUDE_DIR} DIRECTORY)
     list(APPEND LUAROCKS_OPTS
-      --with-lua=${DEPS_INSTALL_DIR}
+      --with-lua=${LUA_ROOT})
-      --with-lua-include=${DEPS_INSTALL_DIR}/include/luajit-2.1
-      --with-lua-interpreter=luajit)
-  elseif(USE_BUNDLED_LUA)
-    list(APPEND LUAROCKS_OPTS
-      --with-lua=${DEPS_INSTALL_DIR})
   else()
-    find_package(Luajit)
+    find_package(Luajit REQUIRED)
-    if(LUAJIT_FOUND)
+    get_filename_component(LUA_ROOT ${LUAJIT_INCLUDE_DIR} DIRECTORY)
-      list(APPEND LUAROCKS_OPTS
+    get_filename_component(LUA_ROOT ${LUA_ROOT} DIRECTORY)
-        --with-lua-include=${LUAJIT_INCLUDE_DIR}
+    list(APPEND LUAROCKS_OPTS
-        --with-lua-interpreter=luajit)
+      --with-lua=${LUA_ROOT}
-    endif()
+      --with-lua-include=${LUAJIT_INCLUDE_DIR}
-
+      --with-lua-interpreter=luajit)
-    # Get LUA_VERSION used with rocks output.
-    if(LUAJIT_FOUND)
-      set(LUA_EXE "luajit")
-    else()
-      set(LUA_EXE "lua")
-    endif()
-    execute_process(
-      COMMAND ${LUA_EXE} -e "print(string.sub(_VERSION, 5))"
-      OUTPUT_VARIABLE LUA_VERSION
-      ERROR_VARIABLE ERR
-      RESULT_VARIABLE RES)
-    if(RES)
-      message(FATAL_ERROR "Could not get LUA_VERSION with ${LUA_EXE}: ${ERR}")
-    endif()
   endif()
 
   set(LUAROCKS_CONFIGURE_COMMAND ${DEPS_BUILD_DIR}/src/luarocks/configure
       --prefix=${DEPS_INSTALL_DIR} --force-config ${LUAROCKS_OPTS})
   set(LUAROCKS_INSTALL_COMMAND ${MAKE_PRG} -j1 bootstrap)
 elseif(MSVC OR MINGW)
-
   if(MINGW)
     set(COMPILER_FLAG /MW)
   elseif(MSVC)
     set(COMPILER_FLAG /MSVC)
   endif()
 
-  # Ignore USE_BUNDLED_LUAJIT - always ON for native Win32
+  find_package(Luajit REQUIRED)
+  # Always assume bundled luajit for native Win32
   set(LUAROCKS_INSTALL_COMMAND install.bat /FORCECONFIG /NOREG /NOADMIN /Q /F
-    /LUA ${DEPS_INSTALL_DIR}
+    /LUA ${DEPS_PREFIX}
-    /LIB ${DEPS_LIB_DIR}
+    /INC ${LUAJIT_INCLUDE_DIR}
-    /BIN ${DEPS_BIN_DIR}
+    /P ${DEPS_INSTALL_DIR}/luarocks
-    /INC ${DEPS_INSTALL_DIR}/include/luajit-2.1
+    /TREE ${DEPS_INSTALL_DIR}
-    /P ${DEPS_INSTALL_DIR}/luarocks /TREE ${DEPS_INSTALL_DIR}
     /SCRIPTS ${DEPS_BIN_DIR}
-    /CMOD ${DEPS_BIN_DIR}
+    ${COMPILER_FLAG})
-    ${COMPILER_FLAG}
-    /LUAMOD ${DEPS_BIN_DIR}/lua)
 
   set(LUAROCKS_BINARY ${DEPS_INSTALL_DIR}/luarocks/luarocks.bat)
 else()
@@ -80,21 +55,17 @@ else()
 endif()
 
 ExternalProject_Add(luarocks
-  URL ${LUAROCKS_URL}
+  URL https://github.com/luarocks/luarocks/archive/v3.9.2.tar.gz
-  URL_HASH SHA256=${LUAROCKS_SHA256}
+  URL_HASH SHA256=a0b36cd68586cd79966d0106bb2e5a4f5523327867995fd66bee4237062b3e3b
   DOWNLOAD_NO_PROGRESS TRUE
   DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/luarocks
   BUILD_IN_SOURCE 1
   CONFIGURE_COMMAND "${LUAROCKS_CONFIGURE_COMMAND}"
   BUILD_COMMAND ""
-  INSTALL_COMMAND "${LUAROCKS_INSTALL_COMMAND}")
+  INSTALL_COMMAND "${LUAROCKS_INSTALL_COMMAND}"
+  EXCLUDE_FROM_ALL TRUE)
 
-if(USE_BUNDLED_LUAJIT)
+set(ROCKS_DIR ${DEPS_LIB_DIR}/luarocks/rocks)
-  add_dependencies(luarocks luajit)
-elseif(USE_BUNDLED_LUA)
-  add_dependencies(luarocks lua)
-endif()
-set(ROCKS_DIR ${DEPS_LIB_DIR}/luarocks/rocks-${LUA_VERSION})
 
 if(MSVC)
   # Workaround for luarocks failing to find the md5sum.exe it is shipped with.
@@ -113,7 +84,7 @@ function(Download ROCK VER)
   add_custom_command(OUTPUT ${OUTPUT}
     COMMAND ${CMAKE_COMMAND} -E env "${PATH}" ${LUAROCKS_BINARY} build ${ROCK} ${VER} ${LUAROCKS_BUILDARGS}
     DEPENDS ${CURRENT_DEP})
-  add_custom_target(${ROCK} ALL DEPENDS ${OUTPUT})
+  add_custom_target(${ROCK} DEPENDS ${OUTPUT})
   set(CURRENT_DEP ${ROCK} PARENT_SCOPE)
 endfunction()
 
@@ -125,9 +96,14 @@ else()
   set(LUACHECK_EXE "${DEPS_BIN_DIR}/luacheck")
 endif()
 
-Download(busted 2.1.1 ${BUSTED_EXE})
+add_custom_target(test_deps)
+
 Download(luacheck 1.1.0-1 ${LUACHECK_EXE})
 
-if (USE_BUNDLED_LUA OR NOT USE_BUNDLED_LUAJIT)
+Download(busted 2.1.1 ${BUSTED_EXE})
+add_dependencies(test_deps busted)
+
+if(PREFER_LUA)
   Download(coxpcall 1.17.0-1)
+  add_dependencies(test_deps coxpcall)
 endif()

cmake/Deps.cmake

@@ -18,3 +18,37 @@ if(APPLE)
 endif()
 
 set(DEPS_CMAKE_CACHE_ARGS -DCMAKE_OSX_ARCHITECTURES:STRING=${CMAKE_OSX_ARCHITECTURES})
+
+# MAKE_PRG
+if(UNIX)
+  find_program(MAKE_PRG NAMES gmake make)
+  if(NOT MAKE_PRG)
+    message(FATAL_ERROR "GNU Make is required to build the dependencies.")
+  else()
+    message(STATUS "Found GNU Make at ${MAKE_PRG}")
+  endif()
+endif()
+# When using make, use the $(MAKE) variable to avoid warning about the job
+# server.
+if(CMAKE_GENERATOR MATCHES "Makefiles")
+  set(MAKE_PRG "$(MAKE)")
+endif()
+if(MINGW AND CMAKE_GENERATOR MATCHES "Ninja")
+  find_program(MAKE_PRG NAMES mingw32-make)
+  if(NOT MAKE_PRG)
+    message(FATAL_ERROR "GNU Make for mingw32 is required to build the dependencies.")
+  else()
+    message(STATUS "Found GNU Make for mingw32: ${MAKE_PRG}")
+  endif()
+endif()
+
+# DEPS_C_COMPILER
+set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}")
+if(CMAKE_OSX_SYSROOT)
+  set(DEPS_C_COMPILER "${DEPS_C_COMPILER} -isysroot${CMAKE_OSX_SYSROOT}")
+endif()
+if(CMAKE_OSX_ARCHITECTURES)
+  foreach(ARCH IN LISTS CMAKE_OSX_ARCHITECTURES)
+    set(DEPS_C_COMPILER "${DEPS_C_COMPILER} -arch ${ARCH}")
+  endforeach()
+endif()

cmake/RunTests.cmake

@@ -63,7 +63,7 @@ if(NOT DEFINED ENV{TEST_TIMEOUT} OR "$ENV{TEST_TIMEOUT}" STREQUAL "")
 endif()
 
 set(ENV{SYSTEM_NAME} ${CMAKE_HOST_SYSTEM_NAME})  # used by test/helpers.lua.
-set(ENV{DEPS_PREFIX} ${DEPS_PREFIX})  # used by test/busted_runner.lua on windows
+set(ENV{DEPS_INSTALL_DIR} ${DEPS_INSTALL_DIR})  # used by test/busted_runner.lua
 
 execute_process(
   COMMAND ${NVIM_PRG} -ll ${WORKING_DIR}/test/busted_runner.lua -v -o test.busted.outputHandlers.${BUSTED_OUTPUT_TYPE}

test/CMakeLists.txt

@@ -20,12 +20,14 @@ if(LUA_HAS_FFI)
       -D BUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
       -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
       -D BUILD_DIR=${CMAKE_BINARY_DIR}
+      -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
       -D TEST_TYPE=unit
       -D CIRRUS_CI=$ENV{CIRRUS_CI}
       -D CI_BUILD=${CI_BUILD}
       -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake
     DEPENDS ${UNITTEST_PREREQS}
     USES_TERMINAL)
+  add_dependencies(unittest test_deps)
 else()
   message(WARNING "disabling unit tests: no Luajit FFI in ${LUA_PRG}")
 endif()
@@ -41,7 +43,7 @@ add_custom_target(functionaltest
     -D BUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
     -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
     -D BUILD_DIR=${CMAKE_BINARY_DIR}
-    -D DEPS_PREFIX=${DEPS_PREFIX}
+    -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
     -D TEST_TYPE=functional
     -D CIRRUS_CI=$ENV{CIRRUS_CI}
     -D CI_BUILD=${CI_BUILD}
@@ -56,6 +58,7 @@ add_custom_target(benchmark
     -D BUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
     -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
     -D BUILD_DIR=${CMAKE_BINARY_DIR}
+    -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
     -D TEST_TYPE=benchmark
     -D CIRRUS_CI=$ENV{CIRRUS_CI}
     -D CI_BUILD=${CI_BUILD}
@@ -70,9 +73,14 @@ add_custom_target(functionaltest-lua
     -D BUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
     -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
     -D BUILD_DIR=${CMAKE_BINARY_DIR}
+    -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
     -D TEST_TYPE=functional
     -D CIRRUS_CI=$ENV{CIRRUS_CI}
     -D CI_BUILD=${CI_BUILD}
     -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake
   DEPENDS ${FUNCTIONALTEST_PREREQS}
   USES_TERMINAL)
+
+add_dependencies(functionaltest test_deps)
+add_dependencies(benchmark test_deps)
+add_dependencies(functionaltest-lua test_deps)

test/busted_runner.lua

@@ -1,12 +1,7 @@
 local platform = vim.loop.os_uname()
-if platform and platform.sysname:lower():find'windows' then
+local deps_install_dir = os.getenv 'DEPS_INSTALL_DIR'
-  local deps_prefix = os.getenv 'DEPS_PREFIX'
+local suffix = (platform and platform.sysname:lower():find'windows') and '.dll' or '.so'
-  if deps_prefix ~= nil and deps_prefix ~= "" then
+package.path = deps_install_dir.."/share/lua/5.1/?.lua;"..deps_install_dir.."/share/lua/5.1/?/init.lua;"..package.path
-    package.path = deps_prefix.."/share/lua/5.1/?.lua;"..deps_prefix.."/share/lua/5.1/?/init.lua;"..package.path
+package.cpath = deps_install_dir.."/lib/lua/5.1/?"..suffix..";"..package.cpath;
-    package.path = deps_prefix.."/bin/lua/?.lua;"..deps_prefix.."/bin/lua/?/init.lua;"..package.path
-    package.cpath = deps_prefix.."/lib/lua/5.1/?.dll;"..package.cpath;
-    package.cpath = deps_prefix.."/bin/?.dll;"..deps_prefix.."/bin/loadall.dll;"..package.cpath;
-  end
-end
 
 require 'busted.runner'({ standalone = false })