refactor(api): make typed dicts appear as types in the source code

problem: can we have Serde?
solution: we have Serde at home

This by itself is just a change of notation, that could be quickly
merged to avoid messy merge conflicts, but upcoming changes are planned:

- keysets no longer need to be defined in one single file. `keysets.h` is
  just the initial automatic conversion of the previous `keysets.lua`.
  keysets just used in a single api/{scope}.h can be moved to that file, later on.

- Typed dicts will have more specific types than Object. this will
  enable most of the existing manual typechecking boilerplate to be eliminated.
  We will need some annotation for missing value, i e a boolean will
  need to be represented as a TriState (none/false/true) in some cases.

- Eventually: optional parameters in form of a `Dict opts` final
  parameter will get added in some form to metadata. this will require
  a discussion/desicion about type forward compatibility.

src/nvim/CMakeLists.txt

@@ -255,7 +255,6 @@ set(GENERATED_UI_EVENTS_METADATA ${GENERATED_DIR}/api/private/ui_events_metadata
 set(GENERATED_EX_CMDS_ENUM ${GENERATED_INCLUDES_DIR}/ex_cmds_enum.generated.h)
 set(GENERATED_EX_CMDS_DEFS ${GENERATED_DIR}/ex_cmds_defs.generated.h)
 set(GENERATED_FUNCS ${GENERATED_DIR}/funcs.generated.h)
-set(GENERATED_KEYSETS ${GENERATED_DIR}/keysets.generated.h)
 set(GENERATED_KEYSETS_DEFS ${GENERATED_DIR}/keysets_defs.generated.h)
 set(GENERATED_EVENTS_ENUM ${GENERATED_INCLUDES_DIR}/auevents_enum.generated.h)
 set(GENERATED_EVENTS_NAMES_MAP ${GENERATED_DIR}/auevents_name_map.generated.h)
@@ -263,7 +262,6 @@ set(GENERATED_OPTIONS ${GENERATED_DIR}/options.generated.h)
 set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua)
 set(FUNCS_GENERATOR ${GENERATOR_DIR}/gen_eval.lua)
 set(EVENTS_GENERATOR ${GENERATOR_DIR}/gen_events.lua)
-set(KEYSETS_GENERATOR ${GENERATOR_DIR}/gen_keysets.lua)
 set(OPTIONS_GENERATOR ${GENERATOR_DIR}/gen_options.lua)
 set(UNICODE_TABLES_GENERATOR ${GENERATOR_DIR}/gen_unicode_tables.lua)
 set(UNICODE_DIR ${PROJECT_SOURCE_DIR}/src/unicode)
@@ -433,7 +431,6 @@ foreach(sfile ${NVIM_SOURCES}
               ${GENERATED_API_DISPATCH}
               "${GENERATED_UI_EVENTS_CALL}"
               "${GENERATED_UI_EVENTS_REMOTE}"
-              "${GENERATED_KEYSETS}"
               "${GENERATED_UI_EVENTS_CLIENT}"
               )
   get_filename_component(full_d ${sfile} DIRECTORY)
@@ -485,11 +482,12 @@ add_custom_command(OUTPUT ${GENERATED_UNICODE_TABLES}
 
 add_custom_command(
   OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_FUNCS_METADATA}
-         ${API_METADATA} ${LUA_API_C_BINDINGS}
+         ${API_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS}
   COMMAND ${LUA_GEN_PRG} ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
                          ${GENERATED_API_DISPATCH}
                          ${GENERATED_FUNCS_METADATA} ${API_METADATA}
                          ${LUA_API_C_BINDINGS}
+                         ${GENERATED_KEYSETS_DEFS}
                          ${API_HEADERS}
   DEPENDS
     ${API_HEADERS}
@@ -560,7 +558,6 @@ list(APPEND NVIM_GENERATED_FOR_SOURCES
   "${GENERATED_API_DISPATCH}"
   "${GENERATED_EX_CMDS_DEFS}"
   "${GENERATED_EVENTS_NAMES_MAP}"
-  "${GENERATED_KEYSETS}"
   "${GENERATED_OPTIONS}"
   "${GENERATED_UNICODE_TABLES}"
   "${VIM_MODULE_FILE}"
@@ -590,12 +587,6 @@ add_custom_command(OUTPUT ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP}
   DEPENDS ${EVENTS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/auevents.lua
 )
 
-add_custom_command(OUTPUT ${GENERATED_KEYSETS} ${GENERATED_KEYSETS_DEFS}
-  COMMAND ${LUA_PRG} ${KEYSETS_GENERATOR}
-      ${CMAKE_CURRENT_LIST_DIR} ${LUA_SHARED_MODULE_SOURCE} ${GENERATED_KEYSETS} ${GENERATED_KEYSETS_DEFS}
-  DEPENDS ${KEYSETS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/api/keysets.lua ${GENERATOR_HASHY}
-)
-
 add_custom_command(OUTPUT ${GENERATED_OPTIONS}
   COMMAND ${LUA_PRG} ${OPTIONS_GENERATOR}
                      ${CMAKE_CURRENT_LIST_DIR} ${GENERATED_OPTIONS}

src/nvim/api/autocmd.h

@@ -3,6 +3,7 @@
 
 #include <stdint.h>
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS

src/nvim/api/buffer.h

@@ -3,6 +3,7 @@
 
 #include <lauxlib.h>
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/buffer_defs.h"
 

src/nvim/api/command.c

@@ -10,6 +10,7 @@
 #include "lauxlib.h"
 #include "nvim/api/command.h"
 #include "nvim/api/private/defs.h"
+#include "nvim/api/private/dispatch.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/api/private/validate.h"
 #include "nvim/ascii.h"

src/nvim/api/command.h

@@ -1,6 +1,7 @@
 #ifndef NVIM_API_COMMAND_H
 #define NVIM_API_COMMAND_H
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/decoration.h"
 #include "nvim/ex_cmds.h"

src/nvim/api/extmark.h

@@ -1,6 +1,7 @@
 #ifndef NVIM_API_EXTMARK_H
 #define NVIM_API_EXTMARK_H
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/decoration.h"
 #include "nvim/macros.h"

src/nvim/api/keysets.h

@@ -0,0 +1,270 @@
+#ifndef NVIM_API_KEYSETS_H
+#define NVIM_API_KEYSETS_H
+
+#include "nvim/api/private/defs.h"
+
+typedef struct {
+  Object types;
+} Dict(context);
+
+typedef struct {
+  Object on_start;
+  Object on_buf;
+  Object on_win;
+  Object on_line;
+  Object on_end;
+  Object _on_hl_def;
+  Object _on_spell_nav;
+} Dict(set_decoration_provider);
+
+typedef struct {
+  Object id;
+  Object end_line;
+  Object end_row;
+  Object end_col;
+  Object hl_group;
+  Object virt_text;
+  Object virt_text_pos;
+  Object virt_text_win_col;
+  Object virt_text_hide;
+  Object hl_eol;
+  Object hl_mode;
+  Object ephemeral;
+  Object priority;
+  Object right_gravity;
+  Object end_right_gravity;
+  Object virt_lines;
+  Object virt_lines_above;
+  Object virt_lines_leftcol;
+  Object strict;
+  Object sign_text;
+  Object sign_hl_group;
+  Object number_hl_group;
+  Object line_hl_group;
+  Object cursorline_hl_group;
+  Object conceal;
+  Object spell;
+  Object ui_watched;
+} Dict(set_extmark);
+
+typedef struct {
+  Object noremap;
+  Object nowait;
+  Object silent;
+  Object script;
+  Object expr;
+  Object unique;
+  Object callback;
+  Object desc;
+  Object replace_keycodes;
+} Dict(keymap);
+
+typedef struct {
+  Object builtin;
+} Dict(get_commands);
+
+typedef struct {
+  Object addr;
+  Object bang;
+  Object bar;
+  Object complete;
+  Object count;
+  Object desc;
+  Object force;
+  Object keepscript;
+  Object nargs;
+  Object preview;
+  Object range;
+  Object register_;
+} Dict(user_command);
+
+typedef struct {
+  Object row;
+  Object col;
+  Object width;
+  Object height;
+  Object anchor;
+  Object relative;
+  Object win;
+  Object bufpos;
+  Object external;
+  Object focusable;
+  Object zindex;
+  Object border;
+  Object title;
+  Object title_pos;
+  Object style;
+  Object noautocmd;
+} Dict(float_config);
+
+typedef struct {
+  Object is_lua;
+  Object do_source;
+} Dict(runtime);
+
+typedef struct {
+  Object winid;
+  Object maxwidth;
+  Object fillchar;
+  Object highlights;
+  Object use_winbar;
+  Object use_tabline;
+  Object use_statuscol_lnum;
+} Dict(eval_statusline);
+
+typedef struct {
+  Object scope;
+  Object win;
+  Object buf;
+  Object filetype;
+} Dict(option);
+
+typedef struct {
+  Object bold;
+  Object standout;
+  Object strikethrough;
+  Object underline;
+  Object undercurl;
+  Object underdouble;
+  Object underdotted;
+  Object underdashed;
+  Object italic;
+  Object reverse;
+  Object altfont;
+  Object nocombine;
+  Object default_;
+  Object cterm;
+  Object foreground;
+  Object fg;
+  Object background;
+  Object bg;
+  Object ctermfg;
+  Object ctermbg;
+  Object special;
+  Object sp;
+  Object link;
+  Object global_link;
+  Object fallback;
+  Object blend;
+  Object fg_indexed;
+  Object bg_indexed;
+} Dict(highlight);
+
+typedef struct {
+  Object bold;
+  Object standout;
+  Object strikethrough;
+  Object underline;
+  Object undercurl;
+  Object underdouble;
+  Object underdotted;
+  Object underdashed;
+  Object italic;
+  Object reverse;
+  Object altfont;
+  Object nocombine;
+} Dict(highlight_cterm);
+
+typedef struct {
+  Object id;
+  Object name;
+  Object link;
+} Dict(get_highlight);
+
+typedef struct {
+  Object buffer;
+  Object event;
+  Object group;
+  Object pattern;
+} Dict(clear_autocmds);
+
+typedef struct {
+  Object buffer;
+  Object callback;
+  Object command;
+  Object desc;
+  Object group;
+  Object nested;
+  Object once;
+  Object pattern;
+} Dict(create_autocmd);
+
+typedef struct {
+  Object buffer;
+  Object group;
+  Object modeline;
+  Object pattern;
+  Object data;
+} Dict(exec_autocmds);
+
+typedef struct {
+  Object event;
+  Object group;
+  Object pattern;
+  Object buffer;
+} Dict(get_autocmds);
+
+typedef struct {
+  Object clear;
+} Dict(create_augroup);
+
+typedef struct {
+  Object cmd;
+  Object range;
+  Object count;
+  Object reg;
+  Object bang;
+  Object args;
+  Object magic;
+  Object mods;
+  Object nargs;
+  Object addr;
+  Object nextcmd;
+} Dict(cmd);
+
+typedef struct {
+  Object file;
+  Object bar;
+} Dict(cmd_magic);
+
+typedef struct {
+  Object silent;
+  Object emsg_silent;
+  Object unsilent;
+  Object filter;
+  Object sandbox;
+  Object noautocmd;
+  Object browse;
+  Object confirm;
+  Object hide;
+  Object horizontal;
+  Object keepalt;
+  Object keepjumps;
+  Object keepmarks;
+  Object keeppatterns;
+  Object lockmarks;
+  Object noswapfile;
+  Object tab;
+  Object verbose;
+  Object vertical;
+  Object split;
+} Dict(cmd_mods);
+
+typedef struct {
+  Object pattern;
+  Object force;
+} Dict(cmd_mods_filter);
+
+typedef struct {
+  Object output;
+} Dict(cmd_opts);
+
+typedef struct {
+  Object verbose;
+} Dict(echo_opts);
+
+typedef struct {
+  Object output;
+} Dict(exec_opts);
+
+#endif  // NVIM_API_KEYSETS_H

src/nvim/api/keysets.lua

@@ -1,239 +0,0 @@
-return {
-  { 'context', {
-    "types";
-  }};
-  { 'set_decoration_provider', {
-    "on_start";
-    "on_buf";
-    "on_win";
-    "on_line";
-    "on_end";
-    "_on_hl_def";
-    "_on_spell_nav";
-  }};
-  { 'set_extmark', {
-    "id";
-    "end_line";
-    "end_row";
-    "end_col";
-    "hl_group";
-    "virt_text";
-    "virt_text_pos";
-    "virt_text_win_col";
-    "virt_text_hide";
-    "hl_eol";
-    "hl_mode";
-    "ephemeral";
-    "priority";
-    "right_gravity";
-    "end_right_gravity";
-    "virt_lines";
-    "virt_lines_above";
-    "virt_lines_leftcol";
-    "strict";
-    "sign_text";
-    "sign_hl_group";
-    "number_hl_group";
-    "line_hl_group";
-    "cursorline_hl_group";
-    "conceal";
-    "spell";
-    "ui_watched";
-  }};
-  { 'keymap', {
-    "noremap";
-    "nowait";
-    "silent";
-    "script";
-    "expr";
-    "unique";
-    "callback";
-    "desc";
-    "replace_keycodes";
-  }};
-  { 'get_commands', {
-    "builtin";
-  }};
-  { 'user_command', {
-    "addr";
-    "bang";
-    "bar";
-    "complete";
-    "count";
-    "desc";
-    "force";
-    "keepscript";
-    "nargs";
-    "preview";
-    "range";
-    "register";
-  }};
-  { 'float_config', {
-    "row";
-    "col";
-    "width";
-    "height";
-    "anchor";
-    "relative";
-    "win";
-    "bufpos";
-    "external";
-    "focusable";
-    "zindex";
-    "border";
-    "title";
-    "title_pos";
-    "style";
-    "noautocmd";
-  }};
-  { 'runtime', {
-    "is_lua";
-    "do_source";
-  }};
-  { 'eval_statusline', {
-    "winid";
-    "maxwidth";
-    "fillchar";
-    "highlights";
-    "use_winbar";
-    "use_tabline";
-    "use_statuscol_lnum";
-  }};
-  { 'option', {
-    "scope";
-    "win";
-    "buf";
-    "filetype";
-  }};
-  { 'highlight', {
-    "bold";
-    "standout";
-    "strikethrough";
-    "underline";
-    "undercurl";
-    "underdouble";
-    "underdotted";
-    "underdashed";
-    "italic";
-    "reverse";
-    "altfont";
-    "nocombine";
-    "default";
-    "cterm";
-    "foreground"; "fg";
-    "background"; "bg";
-    "ctermfg";
-    "ctermbg";
-    "special"; "sp";
-    "link";
-    "global_link";
-    "fallback";
-    "blend";
-    "fg_indexed";
-    "bg_indexed";
-  }};
-  { 'highlight_cterm', {
-    "bold";
-    "standout";
-    "strikethrough";
-    "underline";
-    "undercurl";
-    "underdouble";
-    "underdotted";
-    "underdashed";
-    "italic";
-    "reverse";
-    "altfont";
-    "nocombine";
-  }};
-  { 'get_highlight', {
-    "id";
-    "name";
-    "link";
-  }};
-  -- Autocmds
-  { 'clear_autocmds', {
-    "buffer";
-    "event";
-    "group";
-    "pattern";
-  }};
-  { 'create_autocmd', {
-    "buffer";
-    "callback";
-    "command";
-    "desc";
-    "group";
-    "nested";
-    "once";
-    "pattern";
-  }};
-  { 'exec_autocmds', {
-    "buffer";
-    "group";
-    "modeline";
-    "pattern";
-    "data";
-  }};
-  { 'get_autocmds', {
-    "event";
-    "group";
-    "pattern";
-    "buffer";
-  }};
-  { 'create_augroup', {
-    "clear";
-  }};
-  { 'cmd', {
-    "cmd";
-    "range";
-    "count";
-    "reg";
-    "bang";
-    "args";
-    "magic";
-    "mods";
-    "nargs";
-    "addr";
-    "nextcmd";
-  }};
-  { 'cmd_magic', {
-    "file";
-    "bar";
-  }};
-  { 'cmd_mods', {
-    "silent";
-    "emsg_silent";
-    "unsilent";
-    "filter";
-    "sandbox";
-    "noautocmd";
-    "browse";
-    "confirm";
-    "hide";
-    "horizontal";
-    "keepalt";
-    "keepjumps";
-    "keepmarks";
-    "keeppatterns";
-    "lockmarks";
-    "noswapfile";
-    "tab";
-    "verbose";
-    "vertical";
-    "split";
-  }};
-  { 'cmd_mods_filter', {
-    "pattern";
-    "force";
-  }};
-  { 'cmd_opts', {
-    "output";
-  }};
-  { 'echo_opts', {
-    "verbose";
-  }};
-  { 'exec_opts', {
-    "output";
-  }};
-}

src/nvim/api/options.h

@@ -1,7 +1,9 @@
 #ifndef NVIM_API_OPTIONS_H
 #define NVIM_API_OPTIONS_H
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "api/options.h.generated.h"
 #endif

src/nvim/api/private/defs.h

@@ -130,8 +130,4 @@ typedef struct {
   size_t ptr_off;
 } KeySetLink;
 
-#ifdef INCLUDE_GENERATED_DECLARATIONS
-# include "keysets_defs.generated.h"
-#endif
-
 #endif  // NVIM_API_PRIVATE_DEFS_H

src/nvim/api/private/dispatch.h

@@ -28,6 +28,7 @@ extern const MsgpackRpcRequestHandler method_handlers[];
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "api/private/dispatch.h.generated.h"
 # include "api/private/dispatch_wrappers.h.generated.h"
+# include "keysets_defs.generated.h"
 #endif
 
 #endif  // NVIM_API_PRIVATE_DISPATCH_H

src/nvim/api/private/helpers.h

@@ -181,7 +181,6 @@ typedef struct {
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "api/private/helpers.h.generated.h"
-# include "keysets.h.generated.h"
 #endif
 
 #define WITH_SCRIPT_CONTEXT(channel_id, code) \

src/nvim/api/vim.h

@@ -1,6 +1,7 @@
 #ifndef NVIM_API_VIM_H
 #define NVIM_API_VIM_H
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS

src/nvim/api/vimscript.h

@@ -1,6 +1,7 @@
 #ifndef NVIM_API_VIMSCRIPT_H
 #define NVIM_API_VIMSCRIPT_H
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS

src/nvim/api/win_config.h

@@ -1,6 +1,7 @@
 #ifndef NVIM_API_WIN_CONFIG_H
 #define NVIM_API_WIN_CONFIG_H
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS

src/nvim/generators/c_grammar.lua

@@ -55,5 +55,11 @@ local c_proto = Ct(
   fill * P(';')
   )
 
-local grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1)
+local c_field = Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * P(';') * fill)
+local c_keyset = Ct(
+   P('typedef') * ws * P('struct') * fill * P('{') * fill *
+   Cg(Ct(c_field ^ 1), 'fields') *
+   P('}') * fill * P('Dict') * fill * P('(') * Cg(c_id, 'keyset_name') * fill * P(')') * P(';'))
+
+local grammar = Ct((c_proto + c_comment + c_preproc + ws + c_keyset) ^ 1)
 return {grammar=grammar, typed_container=typed_container}

src/nvim/generators/gen_api_dispatch.lua

@@ -8,6 +8,7 @@ if arg[1] == '--help' then
   print('      3: functions metadata output file (funcs_metadata.generated.h)')
   print('      4: API metadata output file (api_metadata.mpack)')
   print('      5: lua C bindings output file (lua_api_c_bindings.generated.c)')
+  print('      6: keyset definitions output file (keysets_defs.generated.h)')
   print('      rest: C files where API functions are defined')
 end
 assert(#arg >= 4)
@@ -32,6 +33,7 @@ local funcs_metadata_outputf = arg[3]
 -- output metadata mpack file, for use by other build scripts
 local mpack_outputf = arg[4]
 local lua_c_bindings_outputf = arg[5]
+local keysets_outputf = arg[6]
 
 -- set of function names, used to detect duplicates
 local function_names = {}
@@ -42,8 +44,57 @@ local function startswith(String,Start)
   return string.sub(String,1,string.len(Start))==Start
 end
 
+local function add_function(fn)
+  local public = startswith(fn.name, "nvim_") or fn.deprecated_since
+  if public and not fn.noexport then
+    functions[#functions + 1] = fn
+    function_names[fn.name] = true
+    if #fn.parameters >= 2 and fn.parameters[2][1] == 'Array' and fn.parameters[2][2] == 'uidata' then
+      -- function receives the "args" as a parameter
+      fn.receives_array_args = true
+      -- remove the args parameter
+      table.remove(fn.parameters, 2)
+    end
+    if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
+      -- this function should receive the channel id
+      fn.receives_channel_id = true
+      -- remove the parameter since it won't be passed by the api client
+      table.remove(fn.parameters, 1)
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then
+      -- function can fail if the last parameter type is 'Error'
+      fn.can_fail = true
+      -- remove the error parameter, msgpack has it's own special field
+      -- for specifying errors
+      fn.parameters[#fn.parameters] = nil
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'arena' then
+      -- return value is allocated in an arena
+      fn.arena_return = true
+      fn.parameters[#fn.parameters] = nil
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'lstate' then
+      fn.has_lua_imp = true
+      fn.parameters[#fn.parameters] = nil
+    end
+  end
+end
+
+local keysets = {}
+
+local function add_keyset(val)
+  local keys = {}
+  for _,field in ipairs(val.fields) do
+    if field.type ~= 'Object' then
+      error 'not yet implemented: types other than Object'
+    end
+    table.insert(keys, field.name)
+  end
+  table.insert(keysets, {val.keyset_name, keys})
+end
+
 -- read each input file, parse and append to the api metadata
-for i = 6, #arg do
+for i = 7, #arg do
   local full_path = arg[i]
   local parts = {}
   for part in string.gmatch(full_path, '[^/]+') do
@@ -55,39 +106,11 @@ for i = 6, #arg do
 
   local tmp = c_grammar.grammar:match(input:read('*all'))
   for j = 1, #tmp do
-    local fn = tmp[j]
+    local val = tmp[j]
-    local public = startswith(fn.name, "nvim_") or fn.deprecated_since
+    if val.keyset_name then
-    if public and not fn.noexport then
+      add_keyset(val)
-      functions[#functions + 1] = tmp[j]
+    else
-      function_names[fn.name] = true
+      add_function(val)
-      if #fn.parameters >= 2 and fn.parameters[2][1] == 'Array' and fn.parameters[2][2] == 'uidata' then
-        -- function receives the "args" as a parameter
-        fn.receives_array_args = true
-        -- remove the args parameter
-        table.remove(fn.parameters, 2)
-      end
-      if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
-        -- this function should receive the channel id
-        fn.receives_channel_id = true
-        -- remove the parameter since it won't be passed by the api client
-        table.remove(fn.parameters, 1)
-      end
-      if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then
-        -- function can fail if the last parameter type is 'Error'
-        fn.can_fail = true
-        -- remove the error parameter, msgpack has it's own special field
-        -- for specifying errors
-        fn.parameters[#fn.parameters] = nil
-      end
-      if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'arena' then
-        -- return value is allocated in an arena
-        fn.arena_return = true
-        fn.parameters[#fn.parameters] = nil
-      end
-      if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'lstate' then
-        fn.has_lua_imp = true
-        fn.parameters[#fn.parameters] = nil
-      end
     end
   end
   input:close()
@@ -195,6 +218,7 @@ funcs_metadata_output:close()
 -- start building the dispatch wrapper output
 local output = io.open(dispatch_outputf, 'wb')
 
+local keysets_defs = io.open(keysets_outputf, 'wb')
 
 -- ===========================================================================
 -- NEW API FILES MUST GO HERE.
@@ -224,6 +248,52 @@ output:write([[
 
 ]])
 
+for _,keyset in ipairs(keysets) do
+  local name, keys = unpack(keyset)
+  local special = {}
+  local function sanitize(key)
+    if special[key] then
+      return key .. "_"
+    end
+    return key
+  end
+
+  for i = 1,#keys do
+    if vim.endswith(keys[i], "_") then
+      keys[i] = string.sub(keys[i],1, #(keys[i]) - 1)
+      special[keys[i]] = true
+    end
+  end
+  local neworder, hashfun = hashy.hashy_hash(name, keys, function (idx)
+    return name.."_table["..idx.."].str"
+  end)
+
+  keysets_defs:write("extern KeySetLink "..name.."_table[];\n")
+
+  output:write("KeySetLink "..name.."_table[] = {\n")
+  for _, key in ipairs(neworder) do
+    output:write('  {"'..key..'", offsetof(KeyDict_'..name..", "..sanitize(key)..")},\n")
+  end
+    output:write('  {NULL, 0},\n')
+  output:write("};\n\n")
+
+  output:write(hashfun)
+
+  output:write([[
+Object *KeyDict_]]..name..[[_get_field(void *retval, const char *str, size_t len)
+{
+  int hash = ]]..name..[[_hash(str, len);
+  if (hash == -1) {
+    return NULL;
+  }
+
+  return (Object *)((char *)retval + ]]..name..[[_table[hash].ptr_off);
+}
+
+]])
+  keysets_defs:write("#define api_free_keydict_"..name.."(x) api_free_keydict(x, "..name.."_table)\n")
+end
+
 local function real_type(type)
   local rv = type
   local rmatch = string.match(type, "Dict%(([_%w]+)%)")
@@ -475,6 +545,7 @@ output:write([[
 #include "nvim/func_attr.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/private/dispatch.h"
 #include "nvim/lua/converter.h"
 #include "nvim/lua/executor.h"
 #include "nvim/memory.h"
@@ -670,3 +741,4 @@ output:write([[
 ]])
 
 output:close()
+keysets_defs:close()

src/nvim/generators/gen_keysets.lua

@@ -1,80 +0,0 @@
-local nvimsrcdir = arg[1]
-local shared_file = arg[2]
-local funcs_file = arg[3]
-local defs_file = arg[4]
-
-_G.vim = loadfile(shared_file)()
-
-if nvimsrcdir == '--help' then
-  print([[
-Usage:
-  lua gen_keyset.lua TODOFIXUPDATETHIS
-
-Will generate build/src/nvim/auto/keyset.generated.h with definition of functions
-static const array.
-]])
-  os.exit(0)
-end
-
-
-package.path = nvimsrcdir .. '/?.lua;' .. package.path
-local hashy = require'generators.hashy'
-
-local funcspipe = io.open(funcs_file, 'wb')
-local defspipe = io.open(defs_file, 'wb')
-
-local keysets = require'api.keysets'
-
-local keywords = {
-  register = true;
-  default = true;
-}
-
-local function sanitize(key)
-  if keywords[key] then
-    return key .. "_"
-  end
-  return key
-end
-
-for _, v in ipairs(keysets) do
-  local name = v[1]
-  local keys = v[2]
-  local neworder, hashfun = hashy.hashy_hash(name, keys, function (idx)
-    return name.."_table["..idx.."].str"
-  end)
-
-  defspipe:write("typedef struct {\n")
-  for _, key in ipairs(neworder) do
-    defspipe:write("  Object "..sanitize(key)..";\n")
-  end
-  defspipe:write("} KeyDict_"..name..";\n\n")
-
-  defspipe:write("extern KeySetLink "..name.."_table[];\n")
-
-  funcspipe:write("KeySetLink "..name.."_table[] = {\n")
-  for _, key in ipairs(neworder) do
-    funcspipe:write('  {"'..key..'", offsetof(KeyDict_'..name..", "..sanitize(key)..")},\n")
-  end
-    funcspipe:write('  {NULL, 0},\n')
-  funcspipe:write("};\n\n")
-
-  funcspipe:write(hashfun)
-
-  funcspipe:write([[
-Object *KeyDict_]]..name..[[_get_field(void *retval, const char *str, size_t len)
-{
-  int hash = ]]..name..[[_hash(str, len);
-  if (hash == -1) {
-    return NULL;
-  }
-
-  return (Object *)((char *)retval + ]]..name..[[_table[hash].ptr_off);
-}
-
-]])
-  defspipe:write("#define api_free_keydict_"..name.."(x) api_free_keydict(x, "..name.."_table)\n")
-end
-
-funcspipe:close()
-defspipe:close()

src/nvim/highlight.c

@@ -11,6 +11,7 @@
 #include "klib/kvec.h"
 #include "lauxlib.h"
 #include "nvim/api/private/defs.h"
+#include "nvim/api/private/dispatch.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/api/private/validate.h"
 #include "nvim/api/ui.h"

src/nvim/highlight.h

@@ -3,6 +3,7 @@
 
 #include <stdbool.h>
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/buffer_defs.h"
 #include "nvim/highlight_defs.h"

src/nvim/highlight_group.h

@@ -1,6 +1,7 @@
 #ifndef NVIM_HIGHLIGHT_GROUP_H
 #define NVIM_HIGHLIGHT_GROUP_H
 
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/highlight_defs.h"
 #include "nvim/types.h"

src/nvim/mapping.h

@@ -5,6 +5,7 @@
 #include <stddef.h>
 
 #include "lauxlib.h"
+#include "nvim/api/keysets.h"
 #include "nvim/buffer_defs.h"
 #include "nvim/ex_cmds_defs.h"
 #include "nvim/types.h"

src/nvim/msgpack_rpc/helpers.c

@@ -11,6 +11,7 @@
 
 #include "klib/kvec.h"
 #include "msgpack/pack.h"
+#include "nvim/api/keysets.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/assert.h"
 #include "nvim/event/wstream.h"
@@ -19,7 +20,6 @@
 #include "nvim/types.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
-# include "keysets.generated.h"  // IWYU pragma: export
 # include "msgpack_rpc/helpers.c.generated.h"
 #endif