refactor(api): use typed keysets

Initially this is just for geting rid of boilerplate,
but eventually the types could get exposed as metadata

src/nvim/lua/converter.c

@@ -871,7 +871,8 @@ static inline LuaTableProps nlua_check_type(lua_State *const lstate, Error *cons
 {
   if (lua_type(lstate, -1) != LUA_TTABLE) {
     if (err) {
-      api_set_error(err, kErrorTypeValidation, "Expected lua table");
+      api_set_error(err, kErrorTypeValidation, "Expected lua %s",
+                    (type == kObjectTypeFloat) ? "number" : "table");
     }
     return (LuaTableProps) { .type = kObjectTypeNil };
   }
@@ -884,7 +885,7 @@ static inline LuaTableProps nlua_check_type(lua_State *const lstate, Error *cons
 
   if (table_props.type != type) {
     if (err) {
-      api_set_error(err, kErrorTypeValidation, "Unexpected type");
+      api_set_error(err, kErrorTypeValidation, "Expected %s-like lua table", api_typename(type));
     }
   }
 
@@ -1224,26 +1225,19 @@ LuaRef nlua_pop_LuaRef(lua_State *const lstate, Error *err)
   return rv;
 }
 
-#define GENERATE_INDEX_FUNCTION(type) \
-  type nlua_pop_##type(lua_State *lstate, Error *err) \
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT \
-  { \
-    type ret; \
-    if (lua_type(lstate, -1) != LUA_TNUMBER) { \
-      api_set_error(err, kErrorTypeValidation, "Expected Lua number"); \
-      ret = (type) - 1; \
-    } else { \
-      ret = (type)lua_tonumber(lstate, -1); \
-    } \
-    lua_pop(lstate, 1); \
-    return ret; \
+handle_T nlua_pop_handle(lua_State *lstate, Error *err)
+  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  handle_T ret;
+  if (lua_type(lstate, -1) != LUA_TNUMBER) {
+    api_set_error(err, kErrorTypeValidation, "Expected Lua number");
+    ret = (handle_T) - 1;
+  } else {
+    ret = (handle_T)lua_tonumber(lstate, -1);
   }
-
-GENERATE_INDEX_FUNCTION(Buffer)
-GENERATE_INDEX_FUNCTION(Window)
-GENERATE_INDEX_FUNCTION(Tabpage)
-
-#undef GENERATE_INDEX_FUNCTION
+  lua_pop(lstate, 1);
+  return ret;
+}
 
 /// Record some auxiliary values in vim module
 ///
@@ -1292,7 +1286,8 @@ void nlua_init_types(lua_State *const lstate)
   lua_rawset(lstate, -3);
 }
 
-void nlua_pop_keydict(lua_State *L, void *retval, field_hash hashy, Error *err)
+// lua specific variant of api_dict_to_keydict
+void nlua_pop_keydict(lua_State *L, void *retval, FieldHashfn hashy, Error *err)
 {
   if (!lua_istable(L, -1)) {
     api_set_error(err, kErrorTypeValidation, "Expected lua table");
@@ -1305,14 +1300,44 @@ void nlua_pop_keydict(lua_State *L, void *retval, field_hash hashy, Error *err)
     // [dict, key, value]
     size_t len;
     const char *s = lua_tolstring(L, -2, &len);
-    Object *field = hashy(retval, s, len);
+    KeySetLink *field = hashy(s, len);
     if (!field) {
       api_set_error(err, kErrorTypeValidation, "invalid key: %.*s", (int)len, s);
       lua_pop(L, 3);  // []
       return;
     }
 
-    *field = nlua_pop_Object(L, true, err);
+    if (field->opt_index >= 0) {
+      OptKeySet *ks = (OptKeySet *)retval;
+      ks->is_set_ |= (1ULL << field->opt_index);
+    }
+    char *mem = ((char *)retval + field->ptr_off);
+
+    if (field->type == kObjectTypeNil) {
+      *(Object *)mem = nlua_pop_Object(L, true, err);
+    } else if (field->type == kObjectTypeInteger) {
+      *(Integer *)mem = nlua_pop_Integer(L, err);
+    } else if (field->type == kObjectTypeBoolean) {
+      *(Boolean *)mem = nlua_pop_Boolean(L, err);
+    } else if (field->type == kObjectTypeString) {
+      *(String *)mem = nlua_pop_String(L, err);
+    } else if (field->type == kObjectTypeFloat) {
+      *(Float *)mem = nlua_pop_Float(L, err);
+    } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
+               || field->type == kObjectTypeTabpage) {
+      *(handle_T *)mem = nlua_pop_handle(L, err);
+    } else if (field->type == kObjectTypeArray) {
+      *(Array *)mem = nlua_pop_Array(L, err);
+    } else if (field->type == kObjectTypeDictionary) {
+      *(Dictionary *)mem = nlua_pop_Dictionary(L, false, err);
+    } else if (field->type == kObjectTypeLuaRef) {
+      *(LuaRef *)mem = nlua_pop_LuaRef(L, err);
+    } else {
+      abort();
+    }
+    if (ERROR_SET(err)) {
+      break;
+    }
   }
   // [dict]
   lua_pop(L, 1);

src/nvim/lua/converter.h

@@ -9,6 +9,10 @@
 #include "nvim/eval/typval_defs.h"
 #include "nvim/func_attr.h"
 
+#define nlua_pop_Buffer nlua_pop_handle
+#define nlua_pop_Window nlua_pop_handle
+#define nlua_pop_Tabpage nlua_pop_handle
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "lua/converter.h.generated.h"
 #endif