refactor(api): use typed keysets

Initially this is just for geting rid of boilerplate,
but eventually the types could get exposed as metadata

src/nvim/api/private/defs.h

@@ -124,10 +124,20 @@ struct key_value_pair {
   Object value;
 };
 
-typedef Object *(*field_hash)(void *retval, const char *str, size_t len);
+typedef uint64_t OptionalKeys;
+
+// this is the prefix of all keysets with optional keys
+typedef struct {
+  OptionalKeys is_set_;
+} OptKeySet;
+
 typedef struct {
   char *str;
   size_t ptr_off;
+  ObjectType type;  // kObjectTypeNil == untyped
+  int opt_index;
 } KeySetLink;
 
+typedef KeySetLink *(*FieldHashfn)(const char *str, size_t len);
+
 #endif  // NVIM_API_PRIVATE_DEFS_H

src/nvim/api/private/helpers.c

@@ -16,6 +16,7 @@
 #include "nvim/api/private/converter.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/private/validate.h"
 #include "nvim/ascii.h"
 #include "nvim/buffer_defs.h"
 #include "nvim/eval/typval.h"
@@ -915,17 +916,84 @@ free_exit:
   return (HlMessage)KV_INITIAL_VALUE;
 }
 
-bool api_dict_to_keydict(void *rv, field_hash hashy, Dictionary dict, Error *err)
+// see also nlua_pop_keydict for the lua specific implementation
+bool api_dict_to_keydict(void *retval, FieldHashfn hashy, Dictionary dict, Error *err)
 {
   for (size_t i = 0; i < dict.size; i++) {
     String k = dict.items[i].key;
-    Object *field = hashy(rv, k.data, k.size);
+    KeySetLink *field = hashy(k.data, k.size);
     if (!field) {
       api_set_error(err, kErrorTypeValidation, "Invalid key: '%.*s'", (int)k.size, k.data);
       return false;
     }
 
-    *field = dict.items[i].value;
+    if (field->opt_index >= 0) {
+      OptKeySet *ks = (OptKeySet *)retval;
+      ks->is_set_ |= (1ULL << field->opt_index);
+    }
+
+    char *mem = ((char *)retval + field->ptr_off);
+    Object *value = &dict.items[i].value;
+    if (field->type == kObjectTypeNil) {
+      *(Object *)mem = *value;
+    } else if (field->type == kObjectTypeInteger) {
+      VALIDATE_T(field->str, kObjectTypeInteger, value->type, {
+        return false;
+      });
+      *(Integer *)mem = value->data.integer;
+    } else if (field->type == kObjectTypeFloat) {
+      Float *val = (Float *)mem;
+      if (value->type == kObjectTypeInteger) {
+        *val = (Float)value->data.integer;
+      } else {
+        VALIDATE_T(field->str, kObjectTypeFloat, value->type, {
+          return false;
+        });
+        *val = value->data.floating;
+      }
+    } else if (field->type == kObjectTypeBoolean) {
+      // caller should check HAS_KEY to override the nil behavior, or GET_BOOL_OR_TRUE
+      // to directly use true when nil
+      *(Boolean *)mem = api_object_to_bool(*value, field->str, false, err);
+      if (ERROR_SET(err)) {
+        return false;
+      }
+    } else if (field->type == kObjectTypeString) {
+      VALIDATE_T(field->str, kObjectTypeString, value->type, {
+        return false;
+      });
+      *(String *)mem = value->data.string;
+    } else if (field->type == kObjectTypeArray) {
+      VALIDATE_T(field->str, kObjectTypeArray, value->type, {
+        return false;
+      });
+      *(Array *)mem = value->data.array;
+    } else if (field->type == kObjectTypeDictionary) {
+      Dictionary *val = (Dictionary *)mem;
+      // allow empty array as empty dict for lua (directly or via lua-client RPC)
+      if (value->type == kObjectTypeArray && value->data.array.size == 0) {
+        *val = (Dictionary)ARRAY_DICT_INIT;
+      } else if (value->type == kObjectTypeDictionary) {
+        *val = value->data.dictionary;
+      } else {
+        api_err_exp(err, field->str, api_typename(field->type), api_typename(value->type));
+        return false;
+      }
+    } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
+               || field->type == kObjectTypeTabpage) {
+      if (value->type == kObjectTypeInteger || value->type == field->type) {
+        *(handle_T *)mem = (handle_T)value->data.integer;
+      } else {
+        api_err_exp(err, field->str, api_typename(field->type), api_typename(value->type));
+        return false;
+      }
+    } else if (field->type == kObjectTypeLuaRef) {
+      api_set_error(err, kErrorTypeValidation, "Invalid key: '%.*s' is only allowed from Lua",
+                    (int)k.size, k.data);
+      return false;
+    } else {
+      abort();
+    }
   }
 
   return true;
@@ -934,7 +1002,18 @@ bool api_dict_to_keydict(void *rv, field_hash hashy, Dictionary dict, Error *err
 void api_free_keydict(void *dict, KeySetLink *table)
 {
   for (size_t i = 0; table[i].str; i++) {
-    api_free_object(*(Object *)((char *)dict + table[i].ptr_off));
+    char *mem = ((char *)dict + table[i].ptr_off);
+    if (table[i].type == kObjectTypeNil) {
+      api_free_object(*(Object *)mem);
+    } else if (table[i].type == kObjectTypeString) {
+      api_free_string(*(String *)mem);
+    } else if (table[i].type == kObjectTypeArray) {
+      api_free_array(*(Array *)mem);
+    } else if (table[i].type == kObjectTypeDictionary) {
+      api_free_dictionary(*(Dictionary *)mem);
+    } else if (table[i].type == kObjectTypeLuaRef) {
+      api_free_luaref(*(LuaRef *)mem);
+    }
   }
 }
 

src/nvim/api/private/helpers.h

@@ -63,8 +63,9 @@
 #define NIL ((Object)OBJECT_INIT)
 #define NULL_STRING ((String)STRING_INIT)
 
-// currently treat key=vim.NIL as if the key was missing
-#define HAS_KEY(o) ((o).type != kObjectTypeNil)
+#define HAS_KEY(d, typ, key) (((d)->is_set__##typ##_ & (1 << KEYSET_OPTIDX_##typ##__##key)) != 0)
+
+#define GET_BOOL_OR_TRUE(d, typ, key) (HAS_KEY(d, typ, key) ? (d)->key : true)
 
 #define PUT(dict, k, v) \
   kv_push(dict, ((KeyValuePair) { .key = cstr_to_string(k), .value = v }))