Merge pull request #40048 from bfredl/noinstall

fix(build.zig): fixes and improvements

.github/workflows/test.yml

@@ -247,7 +247,8 @@ jobs:
     timeout-minutes: 45
     name: build using zig build (linux)
     env:
-      OPTS: -Doptimize=ReleaseSafe
+      # NB: tests assume there is a chosen install path, but nothing needs to be actually installed
+      OPTS: -Doptimize=ReleaseSafe -Dinstall-path=/usr/
     steps:
       - uses: actions/checkout@v6.0.2
         with:
@@ -258,6 +259,7 @@ jobs:
           version: 0.16.0
       - run: sudo apt-get install -y inotify-tools
 
+      - run: zig fmt --check . # run first, otherwise it will descend into zig-pkg..
       - run: mkdir -p build/ # build/ is used for shared log paths, see global ENV defs
       - run: zig build $OPTS test_nlua0
       - run: zig build $OPTS nvim_bin && ./zig-out/bin/nvim --version
@@ -274,7 +276,7 @@ jobs:
     timeout-minutes: 45
     name: build using zig build (macos 15)
     env:
-      OPTS: # empty
+      OPTS: -Dinstall-path=/usr/
     steps:
       - uses: actions/checkout@v6.0.2
         with:

BUILD.md

@@ -564,6 +564,7 @@ Note that the C++ compiler is explicitly set so that it can be found when the de
    + `zig build functionaltest` to run all functionaltests
    + `zig build functionaltest -- test/functional/autocmd/bufenter_spec.lua` to run the tests in one file
    + `zig build unittest` to run all unittests
+   + `zig build oldtest` to run all oldtests
 
 #### Using system dependencies
 
@@ -575,8 +576,19 @@ See the `prepare` function of [this `PKGBUILD`](https://git.sr.ht/~chinmay/nvim_
 
 ## Cross-compiling
 
-Cross-compilation is not supported, but we collect notes here for reference (improvements welcome).
+Cross-compilation is not fully supported, but we collect notes here for reference (improvements welcome).
 Also relevant for webassembly (WASM) build.
 
-- Set `NVIM_HOST_PRG` so that the docs and tags generation works without
+- cmake: Set `NVIM_HOST_PRG` so that the docs and tags generation works without
   depending on the target binary.
+
+- zig build: often is enabled by just setting -Dtarget, e.g. from a linux host
+
+   zig build -Dtarget=aarch64-macos
+
+  will automatically compilp a host lua for use during build. The
+  "-Dhost={target_string}" option can be used to override the platform for
+  running binaries during compile-time. use "-Dhost=native" to force
+  cross-compiling or "-Dhost=" (empty string) to assume that target binaries
+  can run on the host during the build process (e.g. if target is x86 on a
+  x86_64 system, or if emulation set up with binfmt or similar)

build.zig

@@ -58,7 +58,12 @@ pub fn build(b: *std.Build) !void {
     const modern_unix = is_darwin or os_tag.isBSD() or is_linux;
     const is_wasm = t.cpu.arch == .wasm32;
 
-    const cross_compiling = b.option(bool, "cross", "cross compile") orelse is_wasm;
+    const h = b.graph.host.result;
+    const host = b.option([]const u8, "host", "target for host ") orelse
+        if (target.query.isNative() or h.cpu.arch == t.cpu.arch and h.os.tag == t.os.tag) "" else "native";
+    const cross_compiling = host.len > 0;
+    const target_host = if (cross_compiling) b.resolveTargetQuery(try std.Build.parseTargetQuery(.{ .arch_os_abi = host })) else target;
+
     const emscripten_sysroot = b.option([]const u8, "emscripten-sysroot", "path to emscripten sysroot");
     const emscripten_include = if (emscripten_sysroot) |s|
         std.Build.LazyPath{ .cwd_relative = b.pathJoin(&.{ s, "include" }) }
@@ -77,9 +82,6 @@ pub fn build(b: *std.Build) !void {
         break :blk null;
     } else null;
 
-    // TODO(bfredl): option to set nlua0 target explicitly when cross compiling?
-    const target_host = if (cross_compiling) b.graph.host else target;
-
     // without cross_compiling we like to reuse libluv etc at the same optimize level
     const optimize_host = if (cross_compiling) .ReleaseSafe else optimize;
 
@@ -197,7 +199,6 @@ pub fn build(b: *std.Build) !void {
 
     const nlua0 = try build_lua.build_nlua0(
         b,
-        io,
         target_host,
         optimize_host,
         host_use_luajit,
@@ -283,13 +284,12 @@ pub fn build(b: *std.Build) !void {
 
     const version_lua = gen_config.add("nvim_version.lua", lua_version_info(b));
 
-    var config_str = b.fmt("zig build -Doptimize={s}", .{@tagName(optimize)});
+    // Note: these represent the actually resolved target and host platforms, which are often
+    // more verbose than what needs to be passed in (often -Dhost=native is auto-decteted)
+    // TODO(bfredl): we could include stuff like specific cpu features and os version but eh
+    var config_str = b.fmt("zig build -Doptimize={s} -Dtarget={s}", .{ @tagName(optimize), try t.linuxTriple(b.graph.arena) });
     if (cross_compiling) {
-        config_str = b.fmt("{s} -Dcross -Dtarget={s} (host: {s})", .{
-            config_str,
-            try t.linuxTriple(b.allocator),
-            try b.graph.host.result.linuxTriple(b.allocator),
-        });
+        config_str = b.fmt("{s} -Dhost={s}", .{ config_str, try target_host.result.linuxTriple(b.graph.arena) });
     }
 
     const versiondef_step = b.addConfigHeader(.{
@@ -353,17 +353,19 @@ pub fn build(b: *std.Build) !void {
         .VTERM_TEST_FILE = "test/vterm_test_output", // TODO(bfredl): revisit when porting libvterm tests
     });
 
-    const system_install_path = b.option([]const u8, "install-path", "Install path (for packagers)");
-    const install_path = system_install_path orelse b.install_path;
-    const lib_dir = if (system_install_path) |path| b.fmt("{s}/lib", .{path}) else b.lib_dir;
     _ = gen_config.addCopyFile(sysconfig_step.getOutputFile(), "auto/config.h"); // run_preprocessor() workaronnd
+    //
+    const system_install_path = b.option([]const u8, "install-path", "Install path (for packagers)");
+    const vim_dir, const lib_dir = if (system_install_path) |path| confinstall: {
+        const good_path = try replace_backslashes(b, path);
+        break :confinstall .{ b.fmt("{s}/share/nvim", .{good_path}), b.fmt("{s}/lib/nvim", .{good_path}) };
+    } else .{ "", "" };
 
     _ = gen_config.add("auto/pathdef.h", b.fmt(
-        \\char *default_vim_dir = "{s}/share/nvim";
+        \\char *default_vim_dir = "{s}";
         \\char *default_vimruntime_dir = "";
-        \\char *default_lib_dir = "{s}/nvim";
-        // b.lib_dir is typically b.install_path + "/lib" but may be overridden
-    , .{ try replace_backslashes(b, install_path), try replace_backslashes(b, lib_dir) }));
+        \\char *default_lib_dir = "{s}";
+    , .{ vim_dir, lib_dir }));
 
     const opt_version_string = b.option(
         []const u8,
@@ -456,17 +458,17 @@ pub fn build(b: *std.Build) !void {
 
     const gen_headers, const funcs_data = try gen.nvim_gen_sources(
         b,
-        io,
         nlua0,
         &nvim_sources,
         &nvim_headers,
         &api_headers,
         versiondef_git,
         version_lua,
+        !cross_compiling,
     );
 
     const test_config_step = b.addWriteFiles();
-    _ = test_config_step.add("test/cmakeconfig/paths.lua", try test_config(b, io));
+    _ = test_config_step.add("test/cmakeconfig/paths.lua", try test_config(b));
 
     const test_gen_step = b.step("gen_headers", "debug: output generated headers");
     const config_install = b.addInstallDirectory(.{
@@ -533,7 +535,7 @@ pub fn build(b: *std.Build) !void {
     nvim_mod.addIncludePath(b.path("src"));
     nvim_mod.addIncludePath(gen_config.getDirectory());
     nvim_mod.addIncludePath(gen_headers.getDirectory());
-    try build_lua.add_lua_modules(b, io, t, nvim_mod, lpeg, use_luajit, false, sys_opts);
+    try build_lua.add_lua_modules(b, t, nvim_mod, lpeg, use_luajit, false, sys_opts);
 
     var unit_test_sources = try std.ArrayList([]u8).initCapacity(b.allocator, 10);
     if (support_unittests) {
@@ -827,9 +829,9 @@ fn replace_backslashes(b: *std.Build, input: []const u8) ![]const u8 {
         input;
 }
 
-pub fn test_config(b: *std.Build, io: std.Io) ![]u8 {
+pub fn test_config(b: *std.Build) ![]u8 {
     var buf: [std.fs.max_path_bytes]u8 = std.mem.zeroes([std.fs.max_path_bytes]u8);
-    _ = try b.build_root.handle.realPath(io, &buf);
+    _ = try b.build_root.handle.realPath(b.graph.io, &buf);
     const src_path = std.mem.span(@as([*:0]u8, @ptrCast(&buf)));
 
     // we don't use test/cmakeconfig/paths.lua.in because it contains cmake specific logic
@@ -841,7 +843,7 @@ pub fn test_config(b: *std.Build, io: std.Io) ![]u8 {
         \\M.is_asan = "$ENABLE_ASAN_UBSAN" == "ON"
         \\M.is_zig_build = true
         \\M.vterm_test_file = "test/vterm_test_output"
-        \\M.test_build_dir = "{[bin_dir]s}" -- bull
+        \\M.test_build_dir = _G.nvim_build_dir -- command line arg
         \\M.test_source_path = "{[src_path]s}"
         \\M.test_lua_prg = ""
         \\M.test_luajit_prg = ""
@@ -849,7 +851,7 @@ pub fn test_config(b: *std.Build, io: std.Io) ![]u8 {
         \\M.include_paths = _G.c_include_path or {{}}
         \\
         \\return M
-    , .{ .bin_dir = try replace_backslashes(b, b.install_path), .src_path = try replace_backslashes(b, src_path) });
+    , .{ .src_path = try replace_backslashes(b, src_path) });
 }
 
 fn appendSystemIncludePath(

runtime/doc/news.txt

@@ -131,6 +131,19 @@ BUILD
 
 • Building using "zig build" requires zig 0.16.x.
 
+• zig build: "-Dcross" option was removed. Often cross-compilation is now
+detected, so e.g. from a linux host
+
+   zig build -Dtarget=aarch64-macos
+
+will automatically compile a host Lua for use during build.
+
+The new "-Dhost={target_string}" option can be used to override the used host.
+use "-Dhost=native" to force cross-compiling or "-Dhost=" (empty string) to
+assume that target binaries can run on the host during the build process (e.g.
+if target is x86 on a x86_64 system, or if emulation set up with binfmt or
+similar)
+
 DEFAULTS
 
 • todo

src/build_lua.zig

@@ -4,7 +4,6 @@ const LazyPath = std.Build.LazyPath;
 
 pub fn build_nlua0(
     b: *std.Build,
-    io: std.Io,
     target: std.Build.ResolvedTarget,
     optimize: std.builtin.OptimizeMode,
     use_luajit: bool,
@@ -60,7 +59,7 @@ pub fn build_nlua0(
 
         mod.addIncludePath(b.path("src"));
         mod.addIncludePath(b.path("src/includes_fixmelater"));
-        try add_lua_modules(b, io, target.result, mod, lpeg, use_luajit, true, system_integration_options);
+        try add_lua_modules(b, target.result, mod, lpeg, use_luajit, true, system_integration_options);
     }
 
     // for debugging the nlua0 environment
@@ -84,7 +83,6 @@ pub fn build_nlua0(
 
 pub fn add_lua_modules(
     b: *std.Build,
-    io: std.Io,
     target: std.Target,
     mod: *std.Build.Module,
     lpeg_dep: ?*std.Build.Dependency,
@@ -112,7 +110,7 @@ pub fn add_lua_modules(
         .flags = &flags,
     });
     if (system_integration_options.lpeg) {
-        if (try findLpeg(b, io, target)) |lpeg_lib| {
+        if (try findLpeg(b, target)) |lpeg_lib| {
             mod.addLibraryPath(.{ .cwd_relative = std.fs.path.dirname(lpeg_lib).? });
             mod.addObjectFile(.{ .cwd_relative = lpeg_lib });
         }
@@ -186,7 +184,7 @@ pub fn build_libluv(
     return lib;
 }
 
-fn findLpeg(b: *std.Build, io: std.Io, target: std.Target) !?[]const u8 {
+fn findLpeg(b: *std.Build, target: std.Target) !?[]const u8 {
     const filenames = [_][]const u8{
         "lpeg_a",
         "lpeg",
@@ -207,6 +205,7 @@ fn findLpeg(b: *std.Build, io: std.Io, target: std.Target) !?[]const u8 {
         try paths.append(b.allocator, dir);
         try paths.append(b.allocator, b.fmt("{s}/lua/5.1", .{dir}));
     }
+    const io = b.graph.io;
     for (paths.items) |path| {
         var dir = std.Io.Dir.openDirAbsolute(io, path, .{}) catch continue;
         defer dir.close(io);

src/gen/gen_steps.zig

@@ -5,13 +5,13 @@ pub const SourceItem = struct { name: []u8, api_export: bool };
 
 pub fn nvim_gen_sources(
     b: *std.Build,
-    io: std.Io,
     nlua0: *std.Build.Step.Compile,
     nvim_sources: *std.ArrayList(SourceItem),
     nvim_headers: *std.ArrayList([]u8),
     api_headers: *std.ArrayList(LazyPath),
     versiondef_git: LazyPath,
     version_lua: LazyPath,
+    precompile: bool,
 ) !struct { *std.Build.Step.WriteFile, LazyPath } {
     const gen_headers = b.addWriteFiles();
 
@@ -65,7 +65,7 @@ pub fn nvim_gen_sources(
     {
         const gen_step = b.addRunArtifact(nlua0);
         gen_step.addFileArg(b.path("src/gen/gen_char_blob.lua"));
-        gen_step.addArg("-c");
+        if (precompile) gen_step.addArg("-c");
         _ = gen_header(b, gen_step, "lua/vim_module.generated.h", gen_headers);
         // NB: vim._init_packages and vim.inspect must be be first and second ones
         // respectively, otherwise --luamod-dev won't work properly.
@@ -86,6 +86,7 @@ pub fn nvim_gen_sources(
         }
 
         // Dynamically add all Lua _core/ modules (like CMakeLists.txt does)
+        const io = b.graph.io;
         if (b.build_root.handle.openDir(io, "runtime/lua/vim/_core", .{ .iterate = true })) |core_dir_handle| {
             var core_dir = core_dir_handle;
             defer core_dir.close(io);

test/harness.lua

@@ -1034,6 +1034,8 @@ local function run_test(test, reporter, summary, file_summary)
         end
       end
     end
+    -- check for interrupts
+    vim.wait(0)
   end
 
   test.duration = now_seconds() - start_time
@@ -1315,6 +1317,9 @@ local function parse_args(argv)
     end),
     ['--lpath'] = append_value(opts.lpaths),
     ['--cpath'] = append_value(opts.cpaths),
+    ['--default-path'] = set_nonempty_value('--default-path', function(path)
+      opts.default_path = path
+    end),
   }
 
   local i = 1
@@ -1386,7 +1391,11 @@ local function parse_args(argv)
   end
 
   if #opts.paths == 0 then
-    return nil, 'no test paths provided'
+    if opts.default_path then
+      opts.paths[1] = opts.default_path
+    else
+      return nil, 'no test paths provided'
+    end
   end
 
   return opts

test/run_tests.zig

@@ -10,18 +10,15 @@ pub fn testStep(b: *std.Build, kind: []const u8, nvim_bin: *std.Build.Step.Compi
             test_step.addPrefixedDirectoryArg("-I", path);
         }
     }
+    test_step.addArg(b.fmt("-P{s}", .{b.install_path}));
     test_step.addArg("-v");
     test_step.addArg(b.fmt("--helper=./test/{s}/preload.lua", .{kind}));
     test_step.addArg("--lpath=./src/?.lua");
     test_step.addArg("--lpath=./runtime/lua/?.lua");
     test_step.addArg("--lpath=./?.lua");
     test_step.addPrefixedFileArg("--lpath=", config_dir.path(b, "?.lua")); // FULING: not a real file but works anyway?
-    // TODO(bfredl): look into a TEST_ARGS user hook, TEST_TAG, TEST_FILTER.
-    if (b.args) |args| {
-        test_step.addArgs(args); // accept TEST_FILE as a positional argument
-    } else {
-        test_step.addArg(b.fmt("./test/{s}/", .{kind}));
-    }
+    test_step.addArg(b.fmt("--default-path=./test/{s}", .{kind}));
+    if (b.args) |args| test_step.addArgs(args);
 
     const env = test_step.getEnvMap();
     try env.put("NVIM_TEST", "1");
@@ -30,8 +27,6 @@ pub fn testStep(b: *std.Build, kind: []const u8, nvim_bin: *std.Build.Step.Compi
     try env.put("XDG_CONFIG_HOME", "Xtest_xdg/config");
     try env.put("XDG_DATA_HOME", "Xtest_xdg/share");
     try env.put("XDG_STATE_HOME", "Xtest_xdg/state");
-    try env.put("TMPDIR", b.fmt("{s}/Xtest_tmpdir", .{b.install_path}));
-    try env.put("NVIM_LOG_FILE", b.fmt("{s}/Xtest_nvimlog", .{b.install_path}));
 
     _ = env.swapRemove("NVIM");
     _ = env.swapRemove("XDG_DATA_DIRS");

test/runner.lua

@@ -23,6 +23,12 @@ _G.c_include_path = {}
 while _G.arg[1] and vim.startswith(_G.arg[1], '-I') do
   table.insert(_G.c_include_path, string.sub(table.remove(_G.arg, 1), 3))
 end
+if _G.arg[1] and vim.startswith(_G.arg[1], '-P') then
+  _G.nvim_build_dir = string.sub(table.remove(_G.arg, 1), 3)
+
+  vim.env.TMPDIR = _G.nvim_build_dir .. '/Xtest_tmpdir'
+  vim.env.NVIM_LOG_FILE = _G.nvim_build_dir .. '/Xtest_nvimlog'
+end
 
 local root = repo_root()
 prepend_package_roots({ root, root .. '/test', '.', './test' })