msgpack_rpc: Fix crash in log_server_msg

It appears that used msgpack library is not able to parse back message created 
by msgpack_rpc_from_object() if nesting level is too high, so log_server_msg now 
cares about msgpack_unpack_next() return value. Also error message from 
server_notifications_spec.lua is not readable if something is wrong (though at 
least now it does not crash when parsing deeply nested structures).

log_server_msg() in the test reports

    [msgpack-rpc] nvim -> client(1) [error]        "parse error"

src/nvim/msgpack_rpc/channel.c

@@ -816,20 +816,55 @@ static void decref(Channel *channel)
 #define REQ "[request]      "
 #define RES "[response]     "
 #define NOT "[notification] "
+#define ERR "[error]        "
+
+// Cannot define array with negative offsets, so this one is needed to be added
+// to MSGPACK_UNPACK_\* values.
+#define MUR_OFF 2
+
+static const char *const msgpack_error_messages[] = {
+  [MSGPACK_UNPACK_EXTRA_BYTES + MUR_OFF] = "extra bytes found",
+  [MSGPACK_UNPACK_CONTINUE + MUR_OFF] = "incomplete string",
+  [MSGPACK_UNPACK_PARSE_ERROR + MUR_OFF] = "parse error",
+  [MSGPACK_UNPACK_NOMEM_ERROR + MUR_OFF] = "not enough memory",
+};
 
 static void log_server_msg(uint64_t channel_id,
                            msgpack_sbuffer *packed)
 {
   msgpack_unpacked unpacked;
   msgpack_unpacked_init(&unpacked);
-  msgpack_unpack_next(&unpacked, packed->data, packed->size, NULL);
-  uint64_t type = unpacked.data.via.array.ptr[0].via.u64;
   DLOGN("[msgpack-rpc] nvim -> client(%" PRIu64 ") ", channel_id);
-  log_lock();
+  const msgpack_unpack_return result =
-  FILE *f = open_log_file();
+      msgpack_unpack_next(&unpacked, packed->data, packed->size, NULL);
-  fprintf(f, type ? (type == 1 ? RES : NOT) : REQ);
+  switch (result) {
-  log_msg_close(f, unpacked.data);
+    case MSGPACK_UNPACK_SUCCESS: {
-  msgpack_unpacked_destroy(&unpacked);
+      uint64_t type = unpacked.data.via.array.ptr[0].via.u64;
+      log_lock();
+      FILE *f = open_log_file();
+      fprintf(f, type ? (type == 1 ? RES : NOT) : REQ);
+      log_msg_close(f, unpacked.data);
+      msgpack_unpacked_destroy(&unpacked);
+      break;
+    }
+    case MSGPACK_UNPACK_EXTRA_BYTES:
+    case MSGPACK_UNPACK_CONTINUE:
+    case MSGPACK_UNPACK_PARSE_ERROR:
+    case MSGPACK_UNPACK_NOMEM_ERROR: {
+      log_lock();
+      FILE *f = open_log_file();
+      fprintf(f, ERR);
+      log_msg_close(f, (msgpack_object) {
+          .type = MSGPACK_OBJECT_STR,
+          .via.str = {
+              .ptr = (char *) msgpack_error_messages[result + MUR_OFF],
+              .size = (uint32_t) strlen(
+                  msgpack_error_messages[result + MUR_OFF]),
+          },
+      });
+      break;
+    }
+  }
 }
 
 static void log_client_msg(uint64_t channel_id,

test/functional/api/server_notifications_spec.lua

@@ -41,13 +41,29 @@ describe('notify', function()
     it('does not crash for deeply nested variable', function()
       meths.set_var('l', {})
       local nest_level = 1000
-      meths.command(('call map(range(%u), "extend(g:, {\'l\': [g:l]})")'):format(nest_level))
+      meths.command(('call map(range(%u), "extend(g:, {\'l\': [g:l]})")'):format(nest_level - 1))
-      local ret = {}
-      for i = 1, nest_level do
-        ret = {ret}
-      end
       eval('rpcnotify('..channel..', "event", g:l)')
-      -- eq({'notification', 'event', ret}, next_message())
+      local msg = next_message()
+      eq('notification', msg[1])
+      eq('event', msg[2])
+      local act_ret = msg[3]
+      local act_nest_level = 0
+      while act_ret do
+        if type(act_ret) == 'table' then
+          local cur_act_ret = nil
+          for k, v in pairs(act_ret) do
+            eq(1, k)
+            cur_act_ret = v
+          end
+          if cur_act_ret then
+            act_nest_level = act_nest_level + 1
+          end
+          act_ret = cur_act_ret
+        else
+          eq(nil, act_ret)
+        end
+      end
+      eq(nest_level, act_nest_level)
     end)
   end)
 end)