eval: Return different values when dividing by zero

Fixes #3263

src/nvim/eval.c

@@ -4001,12 +4001,22 @@ eval6 (
        * When either side is a float the result is a float.
        */
       if (use_float) {
-        if (op == '*')
+        if (op == '*') {
           f1 = f1 * f2;
-        else if (op == '/') {
+        } else if (op == '/') {
-          /* We rely on the floating point library to handle divide
+          // Division by zero triggers error from AddressSanitizer
-           * by zero to result in "inf" and not a crash. */
+          f1 = (f2 == 0
-          f1 = f2 != 0 ? f1 / f2 : INFINITY;
+                ? (
+#ifdef NAN
+                    f1 == 0
+                    ? NAN
+                    :
+#endif
+                    (f1 > 0
+                     ? INFINITY
+                     : -INFINITY)
+                )
+                : f1 / f2);
         } else {
           EMSG(_("E804: Cannot use '%' with Float"));
           return FAIL;

test/functional/eval/operators_spec.lua

@@ -0,0 +1,28 @@
+local helpers = require('test.functional.helpers')
+local eq = helpers.eq
+local eval = helpers.eval
+local clear = helpers.clear
+
+describe('Division operator', function()
+  before_each(clear)
+
+  it('returns infinity on {positive}/0.0', function()
+    eq('str2float(\'inf\')', eval('string(1.0/0.0)'))
+    eq('str2float(\'inf\')', eval('string(1.0e-100/0.0)'))
+    eq('str2float(\'inf\')', eval('string(1.0e+100/0.0)'))
+    eq('str2float(\'inf\')', eval('string((1.0/0.0)/0.0)'))
+  end)
+
+  it('returns -infinity on {negative}/0.0', function()
+    eq('-str2float(\'inf\')', eval('string((-1.0)/0.0)'))
+    eq('-str2float(\'inf\')', eval('string((-1.0e-100)/0.0)'))
+    eq('-str2float(\'inf\')', eval('string((-1.0e+100)/0.0)'))
+    eq('-str2float(\'inf\')', eval('string((-1.0/0.0)/0.0)'))
+  end)
+
+  it('returns NaN on 0.0/0.0', function()
+    eq('str2float(\'nan\')', eval('string(0.0/0.0)'))
+    eq('str2float(\'nan\')', eval('string(-(0.0/0.0))'))
+    eq('str2float(\'nan\')', eval('string((-0.0)/0.0)'))
+  end)
+end)