mirrors/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2025-10-09 19:36:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

build: make generated source files reproducible #21586

Browse Source 
Problem:
Build is not reproducible, because generated source files (.c/.h/) are not
deterministic, mostly because Lua pairs() is unordered by design (for security).

https://github.com/LuaJIT/LuaJIT/issues/626#issuecomment-707005671
https://www.lua.org/manual/5.1/manual.html#pdf-next
> The order in which the indices are enumerated is not specified [...]
>
>> The hardening of the VM deliberately randomizes string hashes. This in
>> turn randomizes the iteration order of tables with string keys.

Solution:
- Update the code generation scripts to be deterministic.
    - That is only a partial solution: the exported function
      (funcs_metadata.generated.h) and ui event
      (ui_events_metadata.generated.h) metadata have some mpack'ed
      tables, which are not serialized deterministically.
    - As a workaround, introduce `PRG_GEN_LUA` cmake setting, so you can
      inject a modified build of luajit (with LUAJIT_SECURITY_PRN=0)
      that preserves table order.
    - Longer-term we should change the mpack'ed data structure so it no
      longer uses tables keyed by strings.

Closes #20124

Co-Authored-By: dundargoc <gocdundar@gmail.com>
Co-Authored-By: Arnout Engelen <arnout@bzzt.net>
This commit is contained in:
Arnout Engelen
2023-01-23 10:26:46 +01:00
committed by GitHub
parent da671b21cc
commit cb757f2663
14 changed files with 161 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/nvim/CMakeLists.txt
View File

@@ -570,11 +570,11 @@ add_custom_command(OUTPUT ${GENERATED_UNICODE_TABLES}
add_custom_command(
OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_FUNCS_METADATA}
${API_METADATA} ${LUA_API_C_BINDINGS}
COMMAND ${LUA_PRG} ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
${GENERATED_API_DISPATCH}
${GENERATED_FUNCS_METADATA} ${API_METADATA}
${LUA_API_C_BINDINGS}
${API_HEADERS}
COMMAND ${LUA_GEN_PRG} ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
${GENERATED_API_DISPATCH}
${GENERATED_FUNCS_METADATA} ${API_METADATA}
${LUA_API_C_BINDINGS}
${API_HEADERS}
DEPENDS
${API_HEADERS}
${MSGPACK_RPC_HEADERS}
@@ -619,12 +619,12 @@ add_custom_command(
${GENERATED_UI_EVENTS_REMOTE}
${GENERATED_UI_EVENTS_METADATA}
${GENERATED_UI_EVENTS_CLIENT}
COMMAND ${LUA_PRG} ${API_UI_EVENTS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h
${GENERATED_UI_EVENTS_CALL}
${GENERATED_UI_EVENTS_REMOTE}
${GENERATED_UI_EVENTS_METADATA}
${GENERATED_UI_EVENTS_CLIENT}
COMMAND ${LUA_GEN_PRG} ${API_UI_EVENTS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h
${GENERATED_UI_EVENTS_CALL}
${GENERATED_UI_EVENTS_REMOTE}
${GENERATED_UI_EVENTS_METADATA}
${GENERATED_UI_EVENTS_CLIENT}
DEPENDS
${API_UI_EVENTS_GENERATOR}
${GENERATOR_C_GRAMMAR}

93
src/nvim/api/keysets.lua
View File

@@ -1,8 +1,8 @@
return {
context = {
{ 'context', {
"types";
};
set_decoration_provider = {
}};
{ 'set_decoration_provider', {
"on_start";
"on_buf";
"on_win";
@@ -10,8 +10,8 @@ return {
"on_end";
"_on_hl_def";
"_on_spell_nav";
};
set_extmark = {
}};
{ 'set_extmark', {
"id";
"end_line";
"end_row";
@@ -39,8 +39,8 @@ return {
"conceal";
"spell";
"ui_watched";
};
keymap = {
}};
{ 'keymap', {
"noremap";
"nowait";
"silent";
@@ -50,11 +50,11 @@ return {
"callback";
"desc";
"replace_keycodes";
};
get_commands = {
}};
{ 'get_commands', {
"builtin";
};
user_command = {
}};
{ 'user_command', {
"addr";
"bang";
"bar";
@@ -67,8 +67,8 @@ return {
"preview";
"range";
"register";
};
float_config = {
}};
{ 'float_config', {
"row";
"col";
"width";
@@ -85,25 +85,25 @@ return {
"title_pos";
"style";
"noautocmd";
};
runtime = {
}};
{ 'runtime', {
"is_lua";
"do_source";
};
eval_statusline = {
}};
{ 'eval_statusline', {
"winid";
"maxwidth";
"fillchar";
"highlights";
"use_winbar";
"use_tabline";
};
option = {
}};
{ 'option', {
"scope";
"win";
"buf";
};
highlight = {
}};
{ 'highlight', {
"bold";
"standout";
"strikethrough";
@@ -128,8 +128,8 @@ return {
"blend";
"fg_indexed";
"bg_indexed";
};
highlight_cterm = {
}};
{ 'highlight_cterm', {
"bold";
"standout";
"strikethrough";
@@ -141,15 +141,15 @@ return {
"italic";
"reverse";
"nocombine";
};
}};
-- Autocmds
clear_autocmds = {
{ 'clear_autocmds', {
"buffer";
"event";
"group";
"pattern";
};
create_autocmd = {
}};
{ 'create_autocmd', {
"buffer";
"callback";
"command";
@@ -158,24 +158,24 @@ return {
"nested";
"once";
"pattern";
};
exec_autocmds = {
}};
{ 'exec_autocmds', {
"buffer";
"group";
"modeline";
"pattern";
"data";
};
get_autocmds = {
}};
{ 'get_autocmds', {
"event";
"group";
"pattern";
"buffer";
};
create_augroup = {
}};
{ 'create_augroup', {
"clear";
};
cmd = {
}};
{ 'cmd', {
"cmd";
"range";
"count";
@@ -187,12 +187,12 @@ return {
"nargs";
"addr";
"nextcmd";
};
cmd_magic = {
}};
{ 'cmd_magic', {
"file";
"bar";
};
cmd_mods = {
}};
{ 'cmd_mods', {
"silent";
"emsg_silent";
"unsilent";
@@ -213,16 +213,15 @@ return {
"verbose";
"vertical";
"split";
};
cmd_mods_filter = {
}};
{ 'cmd_mods_filter', {
"pattern";
"force";
};
cmd_opts = {
}};
{ 'cmd_opts', {
"output";
};
echo_opts = {
}};
{ 'echo_opts', {
"verbose";
};
}};
}

20
src/nvim/auevents.lua
View File

@@ -127,10 +127,22 @@ return {
'WinScrolled', -- after a window was scrolled or resized
},
aliases = {
BufCreate = 'BufAdd',
BufRead = 'BufReadPost',
BufWrite = 'BufWritePre',
FileEncoding = 'EncodingChanged',
{
'BufCreate',
'BufAdd'
},
{
'BufRead',
'BufReadPost'
},
{
'BufWrite',
'BufWritePre'
},
{
'FileEncoding',
'EncodingChanged'
},
},
-- List of nvim-specific events or aliases for the purpose of generating
-- syntax file

4
src/nvim/generators/gen_api_dispatch.lua
View File

@@ -426,7 +426,9 @@ for _,fn in ipairs(functions) do
end
remote_fns.redraw = {impl_name="ui_client_redraw", fast=true}
local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", vim.tbl_keys(remote_fns), function (idx)
local names = vim.tbl_keys(remote_fns)
table.sort(names)
local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", names, function (idx)
return "method_handlers["..idx.."].name"
end)

9
src/nvim/generators/gen_eval.lua
View File

@@ -73,14 +73,13 @@ for _,fun in ipairs(metadata) do
end
end
local func_names = vim.tbl_keys(funcs)
table.sort(func_names)
local funcsdata = io.open(funcs_file, 'w')
funcsdata:write(mpack.pack(funcs))
funcsdata:write(mpack.pack(func_names))
funcsdata:close()
local names = vim.tbl_keys(funcs)
local neworder, hashfun = hashy.hashy_hash("find_internal_func", names, function (idx)
local neworder, hashfun = hashy.hashy_hash("find_internal_func", func_names, function (idx)
return "functions["..idx.."].name"
end)
hashpipe:write("static const EvalFuncDef functions[] = {\n")

4
src/nvim/generators/gen_events.lua
View File

@@ -32,7 +32,9 @@ for i, event in ipairs(events) do
end
end
for alias, event in pairs(aliases) do
for _, v in ipairs(aliases) do
local alias = v[1]
local event = v[2]
names_tgt:write(('\n {%u, "%s", EVENT_%s},'):format(#alias, alias, event:upper()))
end

5
src/nvim/generators/gen_keysets.lua
View File

@@ -1,4 +1,3 @@
local nvimsrcdir = arg[1]
local shared_file = arg[2]
local funcs_file = arg[3]
@@ -38,7 +37,9 @@ local function sanitize(key)
return key
end
for name, keys in pairs(keysets) do
for _, v in ipairs(keysets) do
local name = v[1]
local keys = v[2]
local neworder, hashfun = hashy.hashy_hash(name, keys, function (idx)
return name.."_table["..idx.."].str"
end)

6
src/nvim/generators/gen_options.lua
View File

@@ -159,7 +159,7 @@ local dump_option = function(i, o)
if #o.scope == 2 then
pv_name = 'OPT_BOTH(' .. pv_name .. ')'
end
defines['PV_' .. varname:sub(3):upper()] = pv_name
table.insert(defines, { 'PV_' .. varname:sub(3):upper() , pv_name})
w(' .indir=' .. pv_name)
end
if o.enable_if then
@@ -192,7 +192,7 @@ w(' [' .. ('%u'):format(#options.options) .. ']={.fullname=NULL}')
w('};')
w('')
for k, v in pairs(defines) do
w('#define ' .. k .. ' ' .. v)
for _, v in ipairs(defines) do
w('#define ' .. v[1] .. ' ' .. v[2])
end
opt_fd:close()