api/helpers: Use typval_encode.h for vim_to_object

This ought to prevent stack overflow, but I do not see this actually working:
*lua* code crashes with stack overflow when trying to deserialize msgpack from
Neovim, Neovim is fine even if nesting level is increased 100x (though test
becomes very slow); not sure how recursive function may survive this. So it
looks like there are currently only two positive effects:

1. NULL lists are returned as empty (#4596).
2. Functional tests are slightly more fast. Very slightly. Checked for Release
   build for test/functional/eval tests because benchmarking of debug mode is
   not very useful.

src/nvim/api/private/helpers.c

@@ -17,6 +17,13 @@
 #include "nvim/map.h"
 #include "nvim/option.h"
 #include "nvim/option_defs.h"
+#include "nvim/eval/typval_encode.h"
+#include "nvim/lib/kvec.h"
+
+/// Helper structure for vim_to_object
+typedef struct {
+  kvec_t(Object) stack;  ///< Object stack.
+} EncodedData;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "api/private/helpers.c.generated.h"
@@ -310,6 +317,179 @@ void set_option_to(void *to, int type, String name, Object value, Error *err)
   }
 }
 
+#define TYPVAL_ENCODE_ALLOW_SPECIALS false
+
+#define TYPVAL_ENCODE_CONV_NIL() \
+    kv_push(edata->stack, NIL)
+
+#define TYPVAL_ENCODE_CONV_BOOL(num) \
+    kv_push(edata->stack, BOOLEAN_OBJ((Boolean) (num)))
+
+#define TYPVAL_ENCODE_CONV_NUMBER(num) \
+    kv_push(edata->stack, INTEGER_OBJ((Integer) (num)))
+
+#define TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER TYPVAL_ENCODE_CONV_NUMBER
+
+#define TYPVAL_ENCODE_CONV_FLOAT(flt) \
+    kv_push(edata->stack, FLOATING_OBJ((Float) (flt)))
+
+#define TYPVAL_ENCODE_CONV_STRING(str, len) \
+    do { \
+      const size_t len_ = (size_t) (len); \
+      const char *const str_ = (const char *) (str); \
+      assert(len_ == 0 || str_ != NULL); \
+      kv_push(edata->stack, STRING_OBJ(((String) { \
+        .data = xmemdupz((len_?str_:""), len_), \
+        .size = len_ \
+      }))); \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_STR_STRING TYPVAL_ENCODE_CONV_STRING
+
+#define TYPVAL_ENCODE_CONV_EXT_STRING(str, len, type) \
+    TYPVAL_ENCODE_CONV_NIL()
+
+#define TYPVAL_ENCODE_CONV_FUNC(fun) \
+    TYPVAL_ENCODE_CONV_NIL()
+
+#define TYPVAL_ENCODE_CONV_EMPTY_LIST() \
+    kv_push(edata->stack, ARRAY_OBJ(((Array) { .capacity = 0, .size = 0 })))
+
+#define TYPVAL_ENCODE_CONV_EMPTY_DICT() \
+    kv_push(edata->stack, \
+            DICTIONARY_OBJ(((Dictionary) { .capacity = 0, .size = 0 })))
+
+static inline void typval_encode_list_start(EncodedData *const edata,
+                                            const size_t len)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  const Object obj = OBJECT_INIT;
+  kv_push(edata->stack, ARRAY_OBJ(((Array) {
+    .capacity = len,
+    .size = 0,
+    .items = xmalloc(len * sizeof(*obj.data.array.items)),
+  })));
+}
+
+#define TYPVAL_ENCODE_CONV_LIST_START(len) \
+    typval_encode_list_start(edata, (size_t) (len))
+
+static inline void typval_encode_between_list_items(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  Object item = kv_pop(edata->stack);
+  Object *const list = &kv_last(edata->stack);
+  assert(list->type == kObjectTypeArray);
+  assert(list->data.array.size < list->data.array.capacity);
+  list->data.array.items[list->data.array.size++] = item;
+}
+
+#define TYPVAL_ENCODE_CONV_LIST_BETWEEN_ITEMS() \
+    typval_encode_between_list_items(edata)
+
+static inline void typval_encode_list_end(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  typval_encode_between_list_items(edata);
+#ifndef NDEBUG
+  const Object *const list = &kv_last(edata->stack);
+  assert(list->data.array.size == list->data.array.capacity);
+#endif
+}
+
+#define TYPVAL_ENCODE_CONV_LIST_END() \
+    typval_encode_list_end(edata)
+
+static inline void typval_encode_dict_start(EncodedData *const edata,
+                                            const size_t len)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  const Object obj = OBJECT_INIT;
+  kv_push(edata->stack, DICTIONARY_OBJ(((Dictionary) {
+    .capacity = len,
+    .size = 0,
+    .items = xmalloc(len * sizeof(*obj.data.dictionary.items)),
+  })));
+}
+
+#define TYPVAL_ENCODE_CONV_DICT_START(len) \
+    typval_encode_dict_start(edata, (size_t) (len))
+
+#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, kv_pair)
+
+static inline void typval_encode_after_key(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  Object key = kv_pop(edata->stack);
+  Object *const dict = &kv_last(edata->stack);
+  assert(dict->type == kObjectTypeDictionary);
+  assert(dict->data.dictionary.size < dict->data.dictionary.capacity);
+  if (key.type == kObjectTypeString) {
+    dict->data.dictionary.items[dict->data.dictionary.size].key
+        = key.data.string;
+  } else {
+    api_free_object(key);
+    dict->data.dictionary.items[dict->data.dictionary.size].key
+        = STATIC_CSTR_TO_STRING("__INVALID_KEY__");
+  }
+}
+
+#define TYPVAL_ENCODE_CONV_DICT_AFTER_KEY() \
+    typval_encode_after_key(edata)
+
+static inline void typval_encode_between_dict_items(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  Object val = kv_pop(edata->stack);
+  Object *const dict = &kv_last(edata->stack);
+  assert(dict->type == kObjectTypeDictionary);
+  assert(dict->data.dictionary.size < dict->data.dictionary.capacity);
+  dict->data.dictionary.items[dict->data.dictionary.size++].value = val;
+}
+
+#define TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS() \
+    typval_encode_between_dict_items(edata)
+
+static inline void typval_encode_dict_end(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  typval_encode_between_dict_items(edata);
+#ifndef NDEBUG
+  const Object *const dict = &kv_last(edata->stack);
+  assert(dict->data.dictionary.size == dict->data.dictionary.capacity);
+#endif
+}
+
+#define TYPVAL_ENCODE_CONV_DICT_END() \
+    typval_encode_dict_end(edata)
+
+#define TYPVAL_ENCODE_CONV_RECURSE(val, conv_type) \
+    TYPVAL_ENCODE_CONV_NIL()
+
+TYPVAL_ENCODE_DEFINE_CONV_FUNCTIONS(static, object, EncodedData *const, edata)
+
+#undef TYPVAL_ENCODE_CONV_STRING
+#undef TYPVAL_ENCODE_CONV_STR_STRING
+#undef TYPVAL_ENCODE_CONV_EXT_STRING
+#undef TYPVAL_ENCODE_CONV_NUMBER
+#undef TYPVAL_ENCODE_CONV_FLOAT
+#undef TYPVAL_ENCODE_CONV_FUNC
+#undef TYPVAL_ENCODE_CONV_EMPTY_LIST
+#undef TYPVAL_ENCODE_CONV_LIST_START
+#undef TYPVAL_ENCODE_CONV_EMPTY_DICT
+#undef TYPVAL_ENCODE_CONV_NIL
+#undef TYPVAL_ENCODE_CONV_BOOL
+#undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER
+#undef TYPVAL_ENCODE_CONV_DICT_START
+#undef TYPVAL_ENCODE_CONV_DICT_END
+#undef TYPVAL_ENCODE_CONV_DICT_AFTER_KEY
+#undef TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS
+#undef TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK
+#undef TYPVAL_ENCODE_CONV_LIST_END
+#undef TYPVAL_ENCODE_CONV_LIST_BETWEEN_ITEMS
+#undef TYPVAL_ENCODE_CONV_RECURSE
+#undef TYPVAL_ENCODE_ALLOW_SPECIALS
+
 /// Convert a vim object to an `Object` instance, recursively expanding
 /// Arrays/Dictionaries.
 ///
@@ -317,13 +497,12 @@ void set_option_to(void *to, int type, String name, Object value, Error *err)
 /// @return The converted value
 Object vim_to_object(typval_T *obj)
 {
-  Object rv;
+  EncodedData edata = { .stack = KV_INITIAL_VALUE };
-  // We use a lookup table to break out of cyclic references
+  encode_vim_to_object(&edata, obj, "vim_to_object argument");
-  PMap(ptr_t) *lookup = pmap_new(ptr_t)();
+  Object ret = kv_A(edata.stack, 0);
-  rv = vim_to_object_rec(obj, lookup);
+  assert(kv_size(edata.stack) == 1);
-  // Free the table
+  kv_destroy(edata.stack);
-  pmap_free(ptr_t)(lookup);
+  return ret;
-  return rv;
 }
 
 buf_T *find_buffer_by_handle(Buffer buffer, Error *err)
@@ -633,131 +812,6 @@ Object copy_object(Object obj)
   }
 }
 
-/// Recursion helper for the `vim_to_object`. This uses a pointer table
-/// to avoid infinite recursion due to cyclic references
-///
-/// @param obj The source object
-/// @param lookup Lookup table containing pointers to all processed objects
-/// @return The converted value
-static Object vim_to_object_rec(typval_T *obj, PMap(ptr_t) *lookup)
-{
-  Object rv = OBJECT_INIT;
-
-  if (obj->v_type == VAR_LIST || obj->v_type == VAR_DICT) {
-    // Container object, add it to the lookup table
-    if (pmap_has(ptr_t)(lookup, obj)) {
-      // It's already present, meaning we alredy processed it so just return
-      // nil instead.
-      return rv;
-    }
-    pmap_put(ptr_t)(lookup, obj, NULL);
-  }
-
-  switch (obj->v_type) {
-    case VAR_SPECIAL:
-      switch (obj->vval.v_special) {
-        case kSpecialVarTrue:
-        case kSpecialVarFalse: {
-          rv.type = kObjectTypeBoolean;
-          rv.data.boolean = (obj->vval.v_special == kSpecialVarTrue);
-          break;
-        }
-        case kSpecialVarNull: {
-          rv.type = kObjectTypeNil;
-          break;
-        }
-      }
-      break;
-
-    case VAR_STRING:
-      rv.type = kObjectTypeString;
-      rv.data.string = cstr_to_string((char *) obj->vval.v_string);
-      break;
-
-    case VAR_NUMBER:
-      rv.type = kObjectTypeInteger;
-      rv.data.integer = obj->vval.v_number;
-      break;
-
-    case VAR_FLOAT:
-      rv.type = kObjectTypeFloat;
-      rv.data.floating = obj->vval.v_float;
-      break;
-
-    case VAR_LIST:
-      {
-        list_T *list = obj->vval.v_list;
-        listitem_T *item;
-
-        if (list != NULL) {
-          rv.type = kObjectTypeArray;
-          assert(list->lv_len >= 0);
-          rv.data.array.size = (size_t)list->lv_len;
-          rv.data.array.items = xmalloc(rv.data.array.size * sizeof(Object));
-
-          uint32_t i = 0;
-          for (item = list->lv_first; item != NULL; item = item->li_next) {
-            rv.data.array.items[i] = vim_to_object_rec(&item->li_tv, lookup);
-            i++;
-          }
-        }
-      }
-      break;
-
-    case VAR_DICT:
-      {
-        dict_T *dict = obj->vval.v_dict;
-        hashtab_T *ht;
-        uint64_t todo;
-        hashitem_T *hi;
-        dictitem_T *di;
-
-        if (dict != NULL) {
-          ht = &obj->vval.v_dict->dv_hashtab;
-          todo = ht->ht_used;
-          rv.type = kObjectTypeDictionary;
-
-          // Count items
-          rv.data.dictionary.size = 0;
-          for (hi = ht->ht_array; todo > 0; ++hi) {
-            if (!HASHITEM_EMPTY(hi)) {
-              todo--;
-              rv.data.dictionary.size++;
-            }
-          }
-
-          rv.data.dictionary.items =
-            xmalloc(rv.data.dictionary.size * sizeof(KeyValuePair));
-          todo = ht->ht_used;
-          uint32_t i = 0;
-
-          // Convert all
-          for (hi = ht->ht_array; todo > 0; ++hi) {
-            if (!HASHITEM_EMPTY(hi)) {
-              di = dict_lookup(hi);
-              // Convert key
-              rv.data.dictionary.items[i].key =
-                cstr_to_string((char *) hi->hi_key);
-              // Convert value
-              rv.data.dictionary.items[i].value =
-                vim_to_object_rec(&di->di_tv, lookup);
-              todo--;
-              i++;
-            }
-          }
-        }
-      }
-      break;
-
-    case VAR_UNKNOWN:
-    case VAR_FUNC:
-      break;
-  }
-
-  return rv;
-}
-
-
 static void set_option_value_for(char *key,
                                  int numval,
                                  char *stringval,

src/nvim/api/private/helpers.h

@@ -27,6 +27,10 @@
     .type = kObjectTypeInteger, \
     .data.integer = i })
 
+#define FLOATING_OBJ(f) ((Object) { \
+    .type = kObjectTypeFloat, \
+    .data.floating = f })
+
 #define STRING_OBJ(s) ((Object) { \
     .type = kObjectTypeString, \
     .data.string = s })
@@ -61,6 +65,13 @@
 
 #define STATIC_CSTR_AS_STRING(s) ((String) {.data = s, .size = sizeof(s) - 1})
 
+/// Create a new String instance, putting data in allocated memory
+///
+/// @param[in]  s  String to work with. Must be a string literal.
+#define STATIC_CSTR_TO_STRING(s) ((String){ \
+    .data = xmemdupz(s, sizeof(s) - 1), \
+    .size = sizeof(s) - 1 })
+
 // Helpers used by the generated msgpack-rpc api wrappers
 #define api_init_boolean
 #define api_init_integer

src/nvim/eval/encode.c

@@ -379,7 +379,6 @@ int encode_read_from_list(ListReaderState *const state, char *const buf,
       } \
       vim_snprintf(ebuf, ARRAY_SIZE(ebuf), "{E724@%zu}", backref); \
       ga_concat(gap, &ebuf[0]); \
-      return OK; \
     } while (0)
 
 #define TYPVAL_ENCODE_ALLOW_SPECIALS false
@@ -426,7 +425,6 @@ TYPVAL_ENCODE_DEFINE_CONV_FUNCTIONS(, echo, garray_T *const, gap)
         EMSG(_("E724: unable to correctly dump variable " \
                "with self-referencing container")); \
       } \
-      return OK; \
     } while (0)
 
 #undef TYPVAL_ENCODE_ALLOW_SPECIALS
@@ -662,9 +660,8 @@ static inline int convert_to_json_string(garray_T *const gap,
 /// Check whether given key can be used in json_encode()
 ///
 /// @param[in]  tv  Key to check.
-static inline bool check_json_key(const typval_T *const tv)
+bool encode_check_json_key(const typval_T *const tv)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_PURE
-  FUNC_ATTR_ALWAYS_INLINE
 {
   if (tv->v_type == VAR_STRING) {
     return true;
@@ -701,7 +698,7 @@ static inline bool check_json_key(const typval_T *const tv)
 #undef TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK
 #define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, kv_pair) \
     do { \
-      if (!check_json_key(&kv_pair->lv_first->li_tv)) { \
+      if (!encode_check_json_key(&kv_pair->lv_first->li_tv)) { \
         EMSG(_("E474: Invalid key in special dictionary")); \
         goto label; \
       } \

src/nvim/eval/typval_encode.h

@@ -184,10 +184,12 @@ typedef kvec_t(MPConvStackVal) MPConvStack;
 /// @param  copyID_attr  Name of the container attribute that holds copyID.
 ///                      After checking whether value of this attribute is
 ///                      copyID (variable) it is set to copyID.
+/// @param  conv_type  Type of the conversion, @see MPConvStackValType.
 #define _TYPVAL_ENCODE_CHECK_SELF_REFERENCE(val, copyID_attr, conv_type) \
     do { \
       if ((val)->copyID_attr == copyID) { \
         TYPVAL_ENCODE_CONV_RECURSE((val), conv_type); \
+        return OK; \
       } \
       (val)->copyID_attr = copyID; \
     } while (0)

test/functional/api/server_notifications_spec.lua

@@ -3,6 +3,7 @@ local helpers = require('test.functional.helpers')(after_each)
 local eq, clear, eval, execute, nvim, next_message =
   helpers.eq, helpers.clear, helpers.eval, helpers.execute, helpers.nvim,
   helpers.next_message
+local meths = helpers.meths
 
 describe('notify', function()
   local channel
@@ -36,5 +37,17 @@ describe('notify', function()
       eval('rpcnotify(0, "event1", 13, 14, 15)')
       eq({'notification', 'event1', {13, 14, 15}}, next_message())
     end)
+
+    it('does not crash for deeply nested variable', function()
+      meths.set_var('l', {})
+      local nest_level = 100000
+      meths.command(('call map(range(%u), "extend(g:, {\'l\': [g:l]})")'):format(nest_level))
+      local ret = {}
+      for i = 1, nest_level do
+        ret = {ret}
+      end
+      eval('rpcnotify('..channel..', "event", g:l)')
+      -- eq({'notification', 'event', ret}, next_message())
+    end)
   end)
 end)