From 829a5dc83eb87de401256c4907fb153d548c9180 Mon Sep 17 00:00:00 2001 From: bfredl Date: Tue, 28 Oct 2025 11:43:30 +0100 Subject: [PATCH] refactor(build): split out metadata step This was part of an attempt to add "git describe" to zig build without re-building too much. This doesn't yet work as zig upstream changes are are also needed, but I think this was a sensible refactor even in isolation, so breaking it out. "API dispatch" doesn't depend on any ui event stuff nor on version, that was just an accident of the metadata collection being crammed into the same file. Remove "manual invocation" instruction. this is going to bitrot anyway. Both cmake and build.zig allows you to extract the command line of a step, so you can debug it separately. --- src/gen/gen_api_dispatch.lua | 106 +++++------------------------------ src/gen/gen_api_metadata.lua | 83 +++++++++++++++++++++++++++ src/gen/gen_steps.zig | 26 ++++++--- src/nvim/CMakeLists.txt | 40 +++++++++---- 4 files changed, 141 insertions(+), 114 deletions(-) create mode 100644 src/gen/gen_api_metadata.lua diff --git a/src/gen/gen_api_dispatch.lua b/src/gen/gen_api_dispatch.lua index 72df33b970..9ee666b003 100644 --- a/src/gen/gen_api_dispatch.lua +++ b/src/gen/gen_api_dispatch.lua @@ -1,30 +1,21 @@ -- Generates C code to bridge API <=> Lua. --- --- Example (manual) invocation: --- --- make --- cp build/nvim_version.lua src/nvim/ --- cd src/nvim --- nvim -l generators/gen_api_dispatch.lua "../../build/src/nvim/auto/api/private/dispatch_wrappers.generated.h" "../../build/src/nvim/auto/api/private/api_metadata.generated.h" "../../build/funcs_metadata.mpack" "../../build/src/nvim/auto/lua_api_c_bindings.generated.h" "../../build/src/nvim/auto/keysets_defs.generated.h" "../../build/ui_metadata.mpack" "../../build/cmake.config/auto/versiondef_git.h" "./api/autocmd.h" "./api/buffer.h" "./api/command.h" "./api/deprecated.h" "./api/extmark.h" "./api/keysets_defs.h" "./api/options.h" "./api/tabpage.h" "./api/ui.h" "./api/vim.h" "./api/vimscript.h" "./api/win_config.h" "./api/window.h" "../../build/include/api/autocmd.h.generated.h" "../../build/include/api/buffer.h.generated.h" "../../build/include/api/command.h.generated.h" "../../build/include/api/deprecated.h.generated.h" "../../build/include/api/extmark.h.generated.h" "../../build/include/api/options.h.generated.h" "../../build/include/api/tabpage.h.generated.h" "../../build/include/api/ui.h.generated.h" "../../build/include/api/vim.h.generated.h" "../../build/include/api/vimscript.h.generated.h" "../../build/include/api/win_config.h.generated.h" "../../build/include/api/window.h.generated.h" +-- to obtain how the script is invoked, look in build/build.ninja and grep for +-- "gen_api_dispatch.lua" local hashy = require 'gen.hashy' local c_grammar = require('gen.c_grammar') -- output h file with generated dispatch functions (dispatch_wrappers.generated.h) local dispatch_outputf = arg[1] --- output h file with packed metadata (api_metadata.generated.h) -local api_metadata_outputf = arg[2] --- output metadata mpack file, for use by other build scripts (funcs_metadata.mpack) -local mpack_outputf = arg[3] +-- output file with exported functions metadata +local exported_funcs_metadata_outputf = arg[2] +-- output mpack file with raw metadata, for use by gen_eval.lua (funcs_metadata.mpack) +local eval_funcs_metadata_outputf = arg[3] local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c local keysets_outputf = arg[5] -- keysets_defs.generated.h -local ui_metadata_inputf = arg[6] -- ui events metadata -local git_version_inputf = arg[7] -- git version header -local nvim_version_inputf = arg[8] -- nvim version -local dump_bin_array_inputf = arg[9] -local dispatch_deprecated_inputf = arg[10] -local pre_args = 10 -assert(#arg >= pre_args) +local dispatch_deprecated_inputf = arg[6] +local pre_args = 6 +assert(#arg >= 6) local function real_type(type, exported) local ptype = c_grammar.typed_container:match(type) @@ -165,8 +156,6 @@ local function add_keyset(val) } end -local ui_options_text = nil - -- read each input file, parse and append to the api metadata for i = pre_args + 1, #arg do local full_path = arg[i] @@ -190,12 +179,9 @@ for i = pre_args + 1, #arg do end end - ui_options_text = ui_options_text or text:match('ui_ext_names%[][^{]+{([^}]+)}') input:close() end ---- @cast ui_options_text string - --- @generic T: table --- @param orig T --- @return T @@ -303,77 +289,11 @@ for _, f in ipairs(functions) do end end -local ui_options = { 'rgb' } -for x in ui_options_text:gmatch('"([a-z][a-z_]+)"') do - table.insert(ui_options, x) -end +local metadata_output = assert(io.open(exported_funcs_metadata_outputf, 'wb')) +metadata_output:write(vim.mpack.encode(exported_functions)) +metadata_output:close() --- @type integer[] -local version = loadfile(nvim_version_inputf)() -local git_version = io.open(git_version_inputf):read '*a' -local version_build = git_version:match('#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL - -local pieces = {} --- @type string[] - --- Naively using mpack.encode({foo=x, bar=y}) will make the build --- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead -local function fixdict(num) - if num > 15 then - error 'implement more dict codes' - end - pieces[#pieces + 1] = string.char(128 + num) -end - -local function put(item, item2) - table.insert(pieces, vim.mpack.encode(item)) - if item2 ~= nil then - table.insert(pieces, vim.mpack.encode(item2)) - end -end - -fixdict(6) - -put('version') -fixdict(1 + #version) -for _, item in ipairs(version) do - -- NB: all items are mandatory. But any error will be less confusing - -- with placeholder vim.NIL (than invalid mpack data) - local val = item[2] == nil and vim.NIL or item[2] - put(item[1], val) -end -put('build', version_build) - -put('functions', exported_functions) -put('ui_events') -table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all')) -put('ui_options', ui_options) - -put('error_types') -fixdict(2) -put('Exception', { id = 0 }) -put('Validation', { id = 1 }) - -put('types') -local types = - { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } } -fixdict(#types) -for i, item in ipairs(types) do - put(item[1]) - fixdict(2) - put('id', i - 1) - put('prefix', item[2]) -end - -local packed = table.concat(pieces) ---- @type fun(api_metadata: file*, name: string, packed: string) -local dump_bin_array = loadfile(dump_bin_array_inputf)() - --- serialize the API metadata using msgpack and embed into the resulting --- binary for easy querying by clients -local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb')) -dump_bin_array(api_metadata_output, 'packed_api_metadata', packed) -api_metadata_output:close() - -- start building the dispatch wrapper output local output = assert(io.open(dispatch_outputf, 'wb')) @@ -757,7 +677,7 @@ output:close() --- @cast functions {[integer]: gen_api_dispatch.Function, keysets: gen_api_dispatch.Keyset[]} functions.keysets = keysets -local mpack_output = assert(io.open(mpack_outputf, 'wb')) +local mpack_output = assert(io.open(eval_funcs_metadata_outputf, 'wb')) mpack_output:write(vim.mpack.encode(functions)) mpack_output:close() diff --git a/src/gen/gen_api_metadata.lua b/src/gen/gen_api_metadata.lua new file mode 100644 index 0000000000..59824caffe --- /dev/null +++ b/src/gen/gen_api_metadata.lua @@ -0,0 +1,83 @@ +local mpack = vim.mpack + +assert(#arg == 7) +local funcs_metadata_inputf = arg[1] -- exported functions metadata +local ui_metadata_inputf = arg[2] -- ui events metadata +local ui_options_inputf = arg[3] -- for ui options +local git_version_inputf = arg[4] -- git version header +local nvim_version_inputf = arg[5] -- nvim version +local dump_bin_array_inputf = arg[6] +local api_metadata_outputf = arg[7] + +local version = loadfile(nvim_version_inputf)() +local git_version = io.open(git_version_inputf):read '*a' +local version_build = git_version:match('#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL + +local text = io.open(ui_options_inputf):read '*a' +local ui_options_text = text:match('ui_ext_names%[][^{]+{([^}]+)}') +local ui_options = { 'rgb' } +for x in ui_options_text:gmatch('"([a-z][a-z_]+)"') do + table.insert(ui_options, x) +end + +local pieces = {} --- @type string[] + +-- Naively using mpack.encode({foo=x, bar=y}) will make the build +-- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead +local function fixdict(num) + if num > 15 then + error 'implement more dict codes' + end + pieces[#pieces + 1] = string.char(128 + num) +end + +local function put(item, item2) + table.insert(pieces, mpack.encode(item)) + if item2 ~= nil then + table.insert(pieces, mpack.encode(item2)) + end +end + +fixdict(6) + +put('version') +fixdict(1 + #version) +for _, item in ipairs(version) do + -- NB: all items are mandatory. But any error will be less confusing + -- with placeholder vim.NIL (than invalid mpack data) + local val = item[2] == nil and vim.NIL or item[2] + put(item[1], val) +end +put('build', version_build) + +put('functions') +table.insert(pieces, io.open(funcs_metadata_inputf, 'rb'):read('*all')) +put('ui_events') +table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all')) +put('ui_options', ui_options) + +put('error_types') +fixdict(2) +put('Exception', { id = 0 }) +put('Validation', { id = 1 }) + +put('types') +local types = + { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } } +fixdict(#types) +for i, item in ipairs(types) do + put(item[1]) + fixdict(2) + put('id', i - 1) + put('prefix', item[2]) +end + +local packed = table.concat(pieces) +--- @type fun(api_metadata: file*, name: string, packed: string) +local dump_bin_array = loadfile(dump_bin_array_inputf)() + +-- serialize the API metadata using msgpack and embed into the resulting +-- binary for easy querying by clients +local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb')) +dump_bin_array(api_metadata_output, 'packed_api_metadata', packed) +api_metadata_output:close() diff --git a/src/gen/gen_steps.zig b/src/gen/gen_steps.zig index 7d3d9bb8ae..da4c57f430 100644 --- a/src/gen/gen_steps.zig +++ b/src/gen/gen_steps.zig @@ -100,32 +100,40 @@ pub fn nvim_gen_sources( break :ui_step ui_metadata; }; - const funcs_metadata = api_step: { + const eval_funcs_metadata, const exported_funcs_metadata = dispatch_step: { const gen_step = b.addRunArtifact(nlua0); gen_step.addFileArg(b.path("src/gen/gen_api_dispatch.lua")); _ = try gen_header_with_header(b, gen_step, "api/private/dispatch_wrappers.generated.h", nlua0, gen_headers); - _ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers); - const funcs_metadata = gen_step.addOutputFileArg("funcs_metadata.mpack"); + const exported_funcs_metadata = gen_step.addOutputFileArg("exported_funcs_metadata.mpack"); + const eval_funcs_metadata = gen_step.addOutputFileArg("eval_funcs_metadata.mpack"); _ = gen_header(b, gen_step, "lua_api_c_bindings.generated.h", gen_headers); _ = gen_header(b, gen_step, "keysets_defs.generated.h", gen_headers); - gen_step.addFileArg(ui_metadata); - gen_step.addFileArg(versiondef_git); - gen_step.addFileArg(version_lua); - gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua")); gen_step.addFileArg(b.path("src/nvim/api/dispatch_deprecated.lua")); // now follows all .h files with exported functions for (api_headers.items) |h| { gen_step.addFileArg(h); } - break :api_step funcs_metadata; + break :dispatch_step .{ eval_funcs_metadata, exported_funcs_metadata }; }; + { + const gen_step = b.addRunArtifact(nlua0); + gen_step.addFileArg(b.path("src/gen/gen_api_metadata.lua")); + gen_step.addFileArg(exported_funcs_metadata); + gen_step.addFileArg(ui_metadata); + gen_step.addFileArg(b.path("src/nvim/api/ui.h")); + gen_step.addFileArg(versiondef_git); + gen_step.addFileArg(version_lua); + gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua")); + _ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers); + } + const funcs_data = eval_step: { const gen_step = b.addRunArtifact(nlua0); gen_step.addFileArg(b.path("src/gen/gen_eval.lua")); _ = gen_header(b, gen_step, "funcs.generated.h", gen_headers); - gen_step.addFileArg(funcs_metadata); + gen_step.addFileArg(eval_funcs_metadata); const funcs_data = gen_step.addOutputFileArg("funcs_data.mpack"); gen_step.addFileArg(b.path("src/nvim/eval.lua")); break :eval_step funcs_data; diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index d6b5a55ca2..3d5e3e188a 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -285,7 +285,8 @@ endif() # Variables #------------------------------------------------------------------------------- -set(FUNCS_METADATA ${PROJECT_BINARY_DIR}/funcs_metadata.mpack) +set(EVAL_FUNCS_METADATA ${PROJECT_BINARY_DIR}/eval_funcs_metadata.mpack) +set(EXPORTED_FUNCS_METADATA ${PROJECT_BINARY_DIR}/exported_funcs_metadata.mpack) set(UI_METADATA ${PROJECT_BINARY_DIR}/ui_metadata.mpack) set(BINARY_LIB_DIR ${PROJECT_BINARY_DIR}/lib/nvim) set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto) @@ -298,6 +299,7 @@ set(NVIM_RUNTIME_DIR ${PROJECT_SOURCE_DIR}/runtime) # GENERATOR_DIR set(API_DISPATCH_GENERATOR ${GENERATOR_DIR}/gen_api_dispatch.lua) set(API_UI_EVENTS_GENERATOR ${GENERATOR_DIR}/gen_api_ui_events.lua) +set(API_METADATA_GENERATOR ${GENERATOR_DIR}/gen_api_metadata.lua) set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua) set(EVENTS_GENERATOR ${GENERATOR_DIR}/gen_events.lua) set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua) @@ -564,16 +566,13 @@ set(NVIM_VERSION_LUA ${PROJECT_BINARY_DIR}/nvim_version.lua) configure_file(${GENERATOR_DIR}/nvim_version.lua.in ${NVIM_VERSION_LUA}) add_custom_command( - OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_API_METADATA} - ${FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS} + OUTPUT ${GENERATED_API_DISPATCH} ${EXPORTED_FUNCS_METADATA} + ${EVAL_FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS} COMMAND ${LUA_GEN} ${API_DISPATCH_GENERATOR} ${GENERATED_API_DISPATCH} - ${GENERATED_API_METADATA} ${FUNCS_METADATA} + ${EXPORTED_FUNCS_METADATA} ${EVAL_FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS} - ${UI_METADATA} - ${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA} - ${GENERATOR_DIR}/dump_bin_array.lua ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua ${API_HEADERS} @@ -584,11 +583,27 @@ add_custom_command( ${API_DISPATCH_GENERATOR} ${GENERATOR_C_GRAMMAR} ${GENERATOR_HASHY} + ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua +) + +add_custom_command( + OUTPUT ${GENERATED_API_METADATA} + COMMAND ${LUA_GEN} ${API_METADATA_GENERATOR} + ${EXPORTED_FUNCS_METADATA} + ${UI_METADATA} + ${CMAKE_CURRENT_LIST_DIR}/api/ui.h + ${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA} + ${GENERATOR_DIR}/dump_bin_array.lua + ${GENERATED_API_METADATA} + + DEPENDS + ${LUA_GEN_DEPS} + ${API_METADATA_GENERATOR} ${GENERATOR_DIR}/dump_bin_array.lua ${UI_METADATA} + ${EXPORTED_FUNCS_METADATA} ${NVIM_VERSION_LUA} ${NVIM_VERSION_GIT_H} - ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua ) add_custom_command( @@ -654,8 +669,8 @@ add_custom_command(OUTPUT ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS} ) add_custom_command(OUTPUT ${GENERATED_FUNCS} ${FUNCS_DATA} - COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua - DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${FUNCS_METADATA} + COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${EVAL_FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua + DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${EVAL_FUNCS_METADATA} ) add_custom_command(OUTPUT ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP} @@ -690,6 +705,7 @@ list(APPEND NVIM_GENERATED_FOR_SOURCES "${GENERATED_OPTIONS}" "${GENERATED_OPTIONS_MAP}" "${VIM_MODULE_FILE}" + "${GENERATED_API_METADATA}" "${PROJECT_BINARY_DIR}/cmake.config/auto/pathdef.h" ) @@ -960,10 +976,10 @@ add_target(doc-vim ) add_target(doc-eval - COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${FUNCS_METADATA} ${PROJECT_BINARY_DIR}/runtime/doc/tags + COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${EVAL_FUNCS_METADATA} ${PROJECT_BINARY_DIR}/runtime/doc/tags DEPENDS nvim - ${FUNCS_METADATA} + ${EVAL_FUNCS_METADATA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${PROJECT_SOURCE_DIR}/src/nvim/eval.lua ${PROJECT_SOURCE_DIR}/src/nvim/options.lua