encode: Fail to dump NaN and infinity

Thanks to vim/vim#654

src/nvim/eval/encode.c

@@ -7,6 +7,7 @@
 #include <msgpack.h>
 #include <inttypes.h>
 #include <assert.h>
+#include <math.h>
 
 #include "nvim/eval/encode.h"
 #include "nvim/buffer_defs.h"  // vimconv_T
@@ -827,9 +828,23 @@ DEFINE_VIML_CONV_FUNCTIONS(, echo, garray_T *const, gap)
 #undef CONV_FLOAT
 #define CONV_FLOAT(flt) \
     do { \
+      const float_T flt_ = (flt); \
+      switch (fpclassify(flt_)) { \
+        case FP_NAN: { \
+          EMSG(_("E474: Unable to represent NaN value in JSON")); \
+          return FAIL; \
+        } \
+        case FP_INFINITE: { \
+          EMSG(_("E474: Unable to represent infinity in JSON")); \
+          return FAIL; \
+        } \
+        default: { \
           char numbuf[NUMBUFLEN]; \
-      vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", (flt)); \
+          vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", flt_); \
-      ga_concat(gap, numbuf); \
+          ga_concat(gap, (char_u *) numbuf); \
+          break; \
+        } \
+      } \
     } while (0)
 
 /// Last used p_enc value

test/functional/eval/json_functions_spec.lua

@@ -472,6 +472,15 @@ describe('json_encode() function', function()
     eq('1.0e50', eval('json_encode(1.0e50)'))
   end)
 
+  it('fails to dump NaN and infinite values', function()
+    eq('Vim(call):E474: Unable to represent NaN value in JSON',
+       exc_exec('call json_encode(str2float("nan"))'))
+    eq('Vim(call):E474: Unable to represent infinity in JSON',
+       exc_exec('call json_encode(str2float("inf"))'))
+    eq('Vim(call):E474: Unable to represent infinity in JSON',
+       exc_exec('call json_encode(-str2float("inf"))'))
+  end)
+
   it('dumps lists', function()
     eq('[]', funcs.json_encode({}))
     eq('[[]]', funcs.json_encode({{}}))