functests: Test for error conditions

During testing found the following bugs:

1. msgpack-gen.lua script is completely unprepared for Float values either in 
   return type or in arguments. Specifically:

   1. At the time of writing relevant code FLOAT_OBJ did not exist as well as 
      FLOATING_OBJ, but it would be used by msgpack-gen.lua should return type 
      be Float. I added FLOATING_OBJ macros later because did not know that 
      msgpack-gen.lua uses these _OBJ macros, otherwise it would be FLOAT_OBJ.
   2. msgpack-gen.lua should use .data.floating in place of .data.float. But it 
      did not expect that .data subattribute may have name different from 
      lowercased type name.

2. vim_replace_termcodes returned its argument as-is if it receives an empty 
   string (as well as _vim_id*() functions did). But if something in returned 
   argument lives in an allocated memory such action will cause double free: 
   once when freeing arguments, then when freeing return value. It did not cause 
   problems yet because msgpack bindings return empty string as {NULL, 0} and 
   nothing was actually allocated.
3. New code in msgpack-gen.lua popped arguments in reversed order, making lua 
   bindings’ signatures be different from API ones.

src/nvim/api/private/helpers.c

@@ -351,7 +351,7 @@ void set_option_to(void *to, int type, String name, Object value, Error *err)
 #define TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER TYPVAL_ENCODE_CONV_NUMBER
 
 #define TYPVAL_ENCODE_CONV_FLOAT(tv, flt) \
-    kv_push(edata->stack, FLOATING_OBJ((Float)(flt)))
+    kv_push(edata->stack, FLOAT_OBJ((Float)(flt)))
 
 #define TYPVAL_ENCODE_CONV_STRING(tv, str, len) \
     do { \

src/nvim/api/private/helpers.h

@@ -27,7 +27,7 @@
     .type = kObjectTypeInteger, \
     .data.integer = i })
 
-#define FLOATING_OBJ(f) ((Object) { \
+#define FLOAT_OBJ(f) ((Object) { \
     .type = kObjectTypeFloat, \
     .data.floating = f })
 

src/nvim/api/vim.c

@@ -139,7 +139,7 @@ String nvim_replace_termcodes(String str, Boolean from_part, Boolean do_lt,
 {
   if (str.size == 0) {
     // Empty string
-    return str;
+    return (String) { .data = NULL, .size = 0 };
   }
 
   char *ptr = NULL;
@@ -843,7 +843,7 @@ static void write_msg(String message, bool to_err)
 /// @return its argument.
 Object _vim_id(Object obj)
 {
-  return obj;
+  return copy_object(obj);
 }
 
 /// Returns array given as argument
@@ -856,7 +856,7 @@ Object _vim_id(Object obj)
 /// @return its argument.
 Array _vim_id_array(Array arr)
 {
-  return arr;
+  return copy_object(ARRAY_OBJ(arr)).data.array;
 }
 
 /// Returns dictionary given as argument
@@ -869,5 +869,18 @@ Array _vim_id_array(Array arr)
 /// @return its argument.
 Dictionary _vim_id_dictionary(Dictionary dct)
 {
-  return dct;
+  return copy_object(DICTIONARY_OBJ(dct)).data.dictionary;
+}
+
+/// Returns floating-point value given as argument
+///
+/// This API function is used for testing. One should not rely on its presence
+/// in plugins.
+///
+/// @param[in]  flt  Value to return.
+///
+/// @return its argument.
+Float _vim_id_float(Float flt)
+{
+  return flt;
 }

src/nvim/msgpack_rpc/helpers.c

@@ -117,7 +117,7 @@ bool msgpack_rpc_to_object(const msgpack_object *const obj, Object *const arg)
       case MSGPACK_OBJECT_FLOAT: {
         STATIC_ASSERT(sizeof(Float) == sizeof(cur.mobj->via.f64),
                       "Msgpack floating-point size does not match API integer");
-        *cur.aobj = FLOATING_OBJ(cur.mobj->via.f64);
+        *cur.aobj = FLOAT_OBJ(cur.mobj->via.f64);
         break;
       }
 #define STR_CASE(type, attr, obj, dest, conv) \

src/nvim/viml/executor/converter.c

@@ -724,16 +724,15 @@ void nlua_push_Object(lua_State *lstate, const Object obj)
 String nlua_pop_String(lua_State *lstate, Error *err)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
+  if (lua_type(lstate, -1) != LUA_TSTRING) {
+    lua_pop(lstate, 1);
+    api_set_error(err, Validation, "Expected lua string");
+    return (String) { .size = 0, .data = NULL };
+  }
   String ret;
 
   ret.data = (char *)lua_tolstring(lstate, -1, &(ret.size));
-
-  if (ret.data == NULL) {
-    lua_pop(lstate, 1);
-    set_api_error("Expected lua string", err);
-    return (String) { .size = 0, .data = NULL };
-  }
-
+  assert(ret.data != NULL);
   ret.data = xmemdupz(ret.data, ret.size);
   lua_pop(lstate, 1);
 
@@ -746,17 +745,19 @@ String nlua_pop_String(lua_State *lstate, Error *err)
 Integer nlua_pop_Integer(lua_State *lstate, Error *err)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
-  Integer ret = 0;
-
-  if (!lua_isnumber(lstate, -1)) {
+  if (lua_type(lstate, -1) != LUA_TNUMBER) {
     lua_pop(lstate, 1);
-    set_api_error("Expected lua integer", err);
-    return ret;
+    api_set_error(err, Validation, "Expected lua number");
+    return 0;
   }
-  ret = (Integer)lua_tonumber(lstate, -1);
+  const lua_Number n = lua_tonumber(lstate, -1);
   lua_pop(lstate, 1);
-
-  return ret;
+  if (n > (lua_Number)API_INTEGER_MAX || n < (lua_Number)API_INTEGER_MIN
+      || ((lua_Number)((Integer)n)) != n) {
+    api_set_error(err, Exception, "Number is not integral");
+    return 0;
+  }
+  return (Integer)n;
 }
 
 /// Convert lua value to boolean
@@ -765,7 +766,7 @@ Integer nlua_pop_Integer(lua_State *lstate, Error *err)
 Boolean nlua_pop_Boolean(lua_State *lstate, Error *err)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
-  Boolean ret = lua_toboolean(lstate, -1);
+  const Boolean ret = lua_toboolean(lstate, -1);
   lua_pop(lstate, 1);
   return ret;
 }
@@ -784,7 +785,7 @@ static inline LuaTableProps nlua_check_type(lua_State *const lstate,
 {
   if (lua_type(lstate, -1) != LUA_TTABLE) {
     if (err) {
-      set_api_error("Expected lua table", err);
+      api_set_error(err, Validation, "Expected lua table");
     }
     return (LuaTableProps) { .type = kObjectTypeNil };
   }
@@ -797,7 +798,7 @@ static inline LuaTableProps nlua_check_type(lua_State *const lstate,
 
   if (table_props.type != type) {
     if (err) {
-      set_api_error("Unexpected type", err);
+      api_set_error(err, Validation, "Unexpected type");
     }
   }
 
@@ -1050,7 +1051,7 @@ Object nlua_pop_Object(lua_State *const lstate, Error *const err)
         const lua_Number n = lua_tonumber(lstate, -1);
         if (n > (lua_Number)API_INTEGER_MAX || n < (lua_Number)API_INTEGER_MIN
             || ((lua_Number)((Integer)n)) != n) {
-          *cur.obj = FLOATING_OBJ((Float)n);
+          *cur.obj = FLOAT_OBJ((Float)n);
         } else {
           *cur.obj = INTEGER_OBJ((Integer)n);
         }
@@ -1094,7 +1095,7 @@ Object nlua_pop_Object(lua_State *const lstate, Error *const err)
             break;
           }
           case kObjectTypeFloat: {
-            *cur.obj = FLOATING_OBJ((Float)table_props.val);
+            *cur.obj = FLOAT_OBJ((Float)table_props.val);
             break;
           }
           case kObjectTypeNil: {