modeline: Handle version number overflow. #5450

Closes #5449

A file containing the string "vim" followed by a very large number in a modeline
location will trigger an overflow in getdigits() which is called by
chk_modeline() when trying to parse the version number.

Add getdigits_safe(), which does not assert overflows, but reports them to the
caller.

test/functional/eval/modeline_spec.lua

@@ -0,0 +1,19 @@
+local helpers = require('test.functional.helpers')(after_each)
+local clear, execute, write_file = helpers.clear, helpers.execute, helpers.write_file
+local eq, eval = helpers.eq, helpers.eval
+
+describe("modeline", function()
+  local tempfile = helpers.tmpname()
+  before_each(clear)
+
+  after_each(function()
+    os.remove(tempfile)
+  end)
+
+  it('does not crash with a large version number', function()
+    write_file(tempfile, 'vim100000000000000000000000')
+    execute('e! ' .. tempfile)
+
+    eq(2, eval('1+1'))  -- Still alive?
+  end)
+end)