refactor(api): make typed dicts appear as types in the source code

problem: can we have Serde?
solution: we have Serde at home

This by itself is just a change of notation, that could be quickly
merged to avoid messy merge conflicts, but upcoming changes are planned:

- keysets no longer need to be defined in one single file. `keysets.h` is
  just the initial automatic conversion of the previous `keysets.lua`.
  keysets just used in a single api/{scope}.h can be moved to that file, later on.

- Typed dicts will have more specific types than Object. this will
  enable most of the existing manual typechecking boilerplate to be eliminated.
  We will need some annotation for missing value, i e a boolean will
  need to be represented as a TriState (none/false/true) in some cases.

- Eventually: optional parameters in form of a `Dict opts` final
  parameter will get added in some form to metadata. this will require
  a discussion/desicion about type forward compatibility.

src/nvim/generators/gen_api_dispatch.lua

@@ -8,6 +8,7 @@ if arg[1] == '--help' then
   print('      3: functions metadata output file (funcs_metadata.generated.h)')
   print('      4: API metadata output file (api_metadata.mpack)')
   print('      5: lua C bindings output file (lua_api_c_bindings.generated.c)')
+  print('      6: keyset definitions output file (keysets_defs.generated.h)')
   print('      rest: C files where API functions are defined')
 end
 assert(#arg >= 4)
@@ -32,6 +33,7 @@ local funcs_metadata_outputf = arg[3]
 -- output metadata mpack file, for use by other build scripts
 local mpack_outputf = arg[4]
 local lua_c_bindings_outputf = arg[5]
+local keysets_outputf = arg[6]
 
 -- set of function names, used to detect duplicates
 local function_names = {}
@@ -42,8 +44,57 @@ local function startswith(String,Start)
   return string.sub(String,1,string.len(Start))==Start
 end
 
+local function add_function(fn)
+  local public = startswith(fn.name, "nvim_") or fn.deprecated_since
+  if public and not fn.noexport then
+    functions[#functions + 1] = fn
+    function_names[fn.name] = true
+    if #fn.parameters >= 2 and fn.parameters[2][1] == 'Array' and fn.parameters[2][2] == 'uidata' then
+      -- function receives the "args" as a parameter
+      fn.receives_array_args = true
+      -- remove the args parameter
+      table.remove(fn.parameters, 2)
+    end
+    if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
+      -- this function should receive the channel id
+      fn.receives_channel_id = true
+      -- remove the parameter since it won't be passed by the api client
+      table.remove(fn.parameters, 1)
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then
+      -- function can fail if the last parameter type is 'Error'
+      fn.can_fail = true
+      -- remove the error parameter, msgpack has it's own special field
+      -- for specifying errors
+      fn.parameters[#fn.parameters] = nil
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'arena' then
+      -- return value is allocated in an arena
+      fn.arena_return = true
+      fn.parameters[#fn.parameters] = nil
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'lstate' then
+      fn.has_lua_imp = true
+      fn.parameters[#fn.parameters] = nil
+    end
+  end
+end
+
+local keysets = {}
+
+local function add_keyset(val)
+  local keys = {}
+  for _,field in ipairs(val.fields) do
+    if field.type ~= 'Object' then
+      error 'not yet implemented: types other than Object'
+    end
+    table.insert(keys, field.name)
+  end
+  table.insert(keysets, {val.keyset_name, keys})
+end
+
 -- read each input file, parse and append to the api metadata
-for i = 6, #arg do
+for i = 7, #arg do
   local full_path = arg[i]
   local parts = {}
   for part in string.gmatch(full_path, '[^/]+') do
@@ -55,39 +106,11 @@ for i = 6, #arg do
 
   local tmp = c_grammar.grammar:match(input:read('*all'))
   for j = 1, #tmp do
-    local fn = tmp[j]
-    local public = startswith(fn.name, "nvim_") or fn.deprecated_since
-    if public and not fn.noexport then
-      functions[#functions + 1] = tmp[j]
-      function_names[fn.name] = true
-      if #fn.parameters >= 2 and fn.parameters[2][1] == 'Array' and fn.parameters[2][2] == 'uidata' then
-        -- function receives the "args" as a parameter
-        fn.receives_array_args = true
-        -- remove the args parameter
-        table.remove(fn.parameters, 2)
-      end
-      if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
-        -- this function should receive the channel id
-        fn.receives_channel_id = true
-        -- remove the parameter since it won't be passed by the api client
-        table.remove(fn.parameters, 1)
-      end
-      if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then
-        -- function can fail if the last parameter type is 'Error'
-        fn.can_fail = true
-        -- remove the error parameter, msgpack has it's own special field
-        -- for specifying errors
-        fn.parameters[#fn.parameters] = nil
-      end
-      if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'arena' then
-        -- return value is allocated in an arena
-        fn.arena_return = true
-        fn.parameters[#fn.parameters] = nil
-      end
-      if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'lstate' then
-        fn.has_lua_imp = true
-        fn.parameters[#fn.parameters] = nil
-      end
+    local val = tmp[j]
+    if val.keyset_name then
+      add_keyset(val)
+    else
+      add_function(val)
     end
   end
   input:close()
@@ -195,6 +218,7 @@ funcs_metadata_output:close()
 -- start building the dispatch wrapper output
 local output = io.open(dispatch_outputf, 'wb')
 
+local keysets_defs = io.open(keysets_outputf, 'wb')
 
 -- ===========================================================================
 -- NEW API FILES MUST GO HERE.
@@ -224,6 +248,52 @@ output:write([[
 
 ]])
 
+for _,keyset in ipairs(keysets) do
+  local name, keys = unpack(keyset)
+  local special = {}
+  local function sanitize(key)
+    if special[key] then
+      return key .. "_"
+    end
+    return key
+  end
+
+  for i = 1,#keys do
+    if vim.endswith(keys[i], "_") then
+      keys[i] = string.sub(keys[i],1, #(keys[i]) - 1)
+      special[keys[i]] = true
+    end
+  end
+  local neworder, hashfun = hashy.hashy_hash(name, keys, function (idx)
+    return name.."_table["..idx.."].str"
+  end)
+
+  keysets_defs:write("extern KeySetLink "..name.."_table[];\n")
+
+  output:write("KeySetLink "..name.."_table[] = {\n")
+  for _, key in ipairs(neworder) do
+    output:write('  {"'..key..'", offsetof(KeyDict_'..name..", "..sanitize(key)..")},\n")
+  end
+    output:write('  {NULL, 0},\n')
+  output:write("};\n\n")
+
+  output:write(hashfun)
+
+  output:write([[
+Object *KeyDict_]]..name..[[_get_field(void *retval, const char *str, size_t len)
+{
+  int hash = ]]..name..[[_hash(str, len);
+  if (hash == -1) {
+    return NULL;
+  }
+
+  return (Object *)((char *)retval + ]]..name..[[_table[hash].ptr_off);
+}
+
+]])
+  keysets_defs:write("#define api_free_keydict_"..name.."(x) api_free_keydict(x, "..name.."_table)\n")
+end
+
 local function real_type(type)
   local rv = type
   local rmatch = string.match(type, "Dict%(([_%w]+)%)")
@@ -475,6 +545,7 @@ output:write([[
 #include "nvim/func_attr.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/private/dispatch.h"
 #include "nvim/lua/converter.h"
 #include "nvim/lua/executor.h"
 #include "nvim/memory.h"
@@ -670,3 +741,4 @@ output:write([[
 ]])
 
 output:close()
+keysets_defs:close()