modeline: Handle version number overflow. #5450

Closes #5449

A file containing the string "vim" followed by a very large number in a modeline
location will trigger an overflow in getdigits() which is called by
chk_modeline() when trying to parse the version number.

Add getdigits_safe(), which does not assert overflows, but reports them to the
caller.

runtime/doc/options.txt

@@ -497,6 +497,7 @@ For example, to use a modeline only for Vim 7.0:
 To use a modeline for Vim after version 7.2:
 	/* vim>702: set cole=2: */ ~
 There can be no blanks between "vim" and the ":".
+The modeline is ignored if {vers} does not fit in an integer. {Nvim}
 
 
 The number of lines that are checked can be set with the 'modelines' option.

src/nvim/buffer.c

@@ -4509,7 +4509,7 @@ chk_modeline (
   char_u      *e;
   char_u      *linecopy;                /* local copy of any modeline found */
   int prev;
-  int vers;
+  intmax_t vers;
   int end;
   int retval = OK;
   char_u      *save_sourcing_name;
@@ -4528,7 +4528,10 @@ chk_modeline (
           e = s + 4;
         else
           e = s + 3;
-        vers = getdigits_int(&e);
+        if (getdigits_safe(&e, &vers) != OK) {
+          continue;
+        }
+
         if (*e == ':'
             && (s[0] != 'V'
                 || STRNCMP(skipwhite(e + 1), "set", 3) == 0)
@@ -4536,10 +4539,11 @@ chk_modeline (
                 || (VIM_VERSION_100 >= vers && isdigit(s[3]))
                 || (VIM_VERSION_100 < vers && s[3] == '<')
                 || (VIM_VERSION_100 > vers && s[3] == '>')
-                || (VIM_VERSION_100 == vers && s[3] == '=')))
+                || (VIM_VERSION_100 == vers && s[3] == '='))) {
           break;
         }
       }
+    }
     prev = *s;
   }
 

src/nvim/charset.c

@@ -1732,6 +1732,26 @@ char_u* skiptowhite_esc(char_u *p) {
   return p;
 }
 
+/// Get a number from a string and skip over it, signalling overflows
+///
+/// @param[out]  pp  A pointer to a pointer to char_u.
+///                  It will be advanced past the read number.
+/// @param[out]  nr  Number read from the string.
+///
+/// @return OK on success, FAIL on error/overflow
+int getdigits_safe(char_u **pp, intmax_t *nr)
+{
+  errno = 0;
+  *nr = strtoimax((char *)(*pp), (char **)pp, 10);
+
+  if ((*nr == INTMAX_MIN || *nr == INTMAX_MAX)
+      && errno == ERANGE) {
+    return FAIL;
+  }
+
+  return OK;
+}
+
 /// Get a number from a string and skip over it.
 ///
 /// @param[out]  pp  A pointer to a pointer to char_u.
@@ -1740,17 +1760,16 @@ char_u* skiptowhite_esc(char_u *p) {
 /// @return Number read from the string.
 intmax_t getdigits(char_u **pp)
 {
-  errno = 0;
-  intmax_t number = strtoimax((char *)*pp, (char **)pp, 10);
-  if (number == INTMAX_MAX || number == INTMAX_MIN) {
-    assert(errno != ERANGE);
-  }
+  intmax_t number;
+  int ret = getdigits_safe(pp, &number);
+
+  (void)ret;  // Avoid "unused variable" warning in Release build
+  assert(ret == OK);
+
   return number;
 }
 
-/// Get an int number from a string.
-///
-/// A getdigits wrapper restricted to int values.
+/// Get an int number from a string. Like getdigits(), but restricted to `int`.
 int getdigits_int(char_u **pp)
 {
   intmax_t number = getdigits(pp);
@@ -1760,9 +1779,7 @@ int getdigits_int(char_u **pp)
   return (int)number;
 }
 
-/// Get a long number from a string.
-///
-/// A getdigits wrapper restricted to long values.
+/// Get a long number from a string. Like getdigits(), but restricted to `long`.
 long getdigits_long(char_u **pp)
 {
   intmax_t number = getdigits(pp);

test/functional/eval/modeline_spec.lua

@@ -0,0 +1,19 @@
+local helpers = require('test.functional.helpers')(after_each)
+local clear, execute, write_file = helpers.clear, helpers.execute, helpers.write_file
+local eq, eval = helpers.eq, helpers.eval
+
+describe("modeline", function()
+  local tempfile = helpers.tmpname()
+  before_each(clear)
+
+  after_each(function()
+    os.remove(tempfile)
+  end)
+
+  it('does not crash with a large version number', function()
+    write_file(tempfile, 'vim100000000000000000000000')
+    execute('e! ' .. tempfile)
+
+    eq(2, eval('1+1'))  -- Still alive?
+  end)
+end)