Merge pull request #12376 from erw7/fix-stack-overflow-on-input-enqueue

input: fix stack overflow

src/nvim/keymap.c

@@ -517,8 +517,8 @@ char_u *get_special_key_name(int c, int modifiers)
 /// @param[in,out]  srcp  Source from which <> are translated. Is advanced to
 ///                       after the <> name if there is a match.
 /// @param[in]  src_len  Length of the srcp.
-/// @param[out]  dst  Location where translation result will be kept. Must have
+/// @param[out]  dst  Location where translation result will be kept. It must
-///                   at least six bytes.
+//                    be at least 19 bytes per "<x>" form.
 /// @param[in]  keycode  Prefer key code, e.g. K_DEL in place of DEL.
 /// @param[in]  in_string  Inside a double quoted string
 ///

src/nvim/os/input.c

@@ -188,8 +188,13 @@ size_t input_enqueue(String keys)
   char *ptr = keys.data;
   char *end = ptr + keys.size;
 
-  while (rbuffer_space(input_buffer) >= 6 && ptr < end) {
+  while (rbuffer_space(input_buffer) >= 19 && ptr < end) {
-    uint8_t buf[6] = { 0 };
+    // A "<x>" form occupies at least 1 characters, and produces up
+    // to 19 characters (1 + 5 * 3 for the char and 3 for a modifier).
+    // In the case of K_SPECIAL(0x80) or CSI(0x9B), 3 bytes are escaped and
+    // needed, but since the keys are UTF-8, so the first byte cannot be
+    // K_SPECIAL(0x80) or CSI(0x9B).
+    uint8_t buf[19] = { 0 };
     unsigned int new_size
         = trans_special((const uint8_t **)&ptr, (size_t)(end - ptr), buf, true,
                         false);

test/functional/api/vim_spec.lua

@@ -482,6 +482,11 @@ describe('API', function()
       eq(true, status)        -- nvim_input() did not fail.
       eq("E117:", v_errnum)   -- v:errmsg was updated.
     end)
+
+    it('does not crash even if trans_special result is largest #11788, #12287', function()
+      command("call nvim_input('<M-'.nr2char(0x40000000).'>')")
+      eq(1, eval('1'))
+    end)
   end)
 
   describe('nvim_paste', function()