fix(lua): avoid recursive vim.on_key() callback (#30753)

runtime/doc/lua.txt

@@ -1678,6 +1678,8 @@ vim.on_key({fn}, {ns_id})                                       *vim.on_key()*
 
     Note: ~
       • {fn} will be removed on error.
+      • {fn} won't be invoked recursively, i.e. if {fn} itself consumes input,
+        it won't be invoked for those keys.
       • {fn} will not be cleared by |nvim_buf_clear_namespace()|
 
     Parameters: ~

runtime/doc/news.txt

@@ -262,6 +262,9 @@ These existing features changed their behavior.
   more emoji characters than before, including those encoded with multiple
   emoji codepoints combined with ZWJ (zero width joiner) codepoints.
 
+• |vim.on_key()| callbacks won't be invoked recursively when a callback itself
+  consumes input.
+
 ==============================================================================
 REMOVED FEATURES                                                 *news-removed*
 

runtime/lua/vim/_editor.lua

@@ -658,6 +658,8 @@ local on_key_cbs = {} --- @type table<integer,function>
 --- and cannot be toggled dynamically.
 ---
 ---@note {fn} will be removed on error.
+---@note {fn} won't be invoked recursively, i.e. if {fn} itself consumes input,
+---           it won't be invoked for those keys.
 ---@note {fn} will not be cleared by |nvim_buf_clear_namespace()|
 ---
 ---@param fn fun(key: string, typed: string)?

src/nvim/lua/executor.c

@@ -2065,6 +2065,13 @@ char *nlua_register_table_as_callable(const typval_T *const arg)
 
 void nlua_execute_on_key(int c, char *typed_buf)
 {
+  static bool recursive = false;
+
+  if (recursive) {
+    return;
+  }
+  recursive = true;
+
   char buf[MB_MAXBYTES * 3 + 4];
   size_t buf_len = special_to_buf(c, mod_mask, false, buf);
   vim_unescape_ks(typed_buf);
@@ -2103,6 +2110,8 @@ void nlua_execute_on_key(int c, char *typed_buf)
   // [ ]
   assert(top == lua_gettop(lstate));
 #endif
+
+  recursive = false;
 }
 
 // Sets the editor "script context" during Lua execution. Used by :verbose.

test/functional/lua/vim_spec.lua

@@ -3223,6 +3223,45 @@ describe('lua stdlib', function()
       feed('<C-C>')
       eq('/', exec_lua([[return _G.ctrl_c_cmdtype]]))
     end)
+
+    it('callback is not invoked recursively #30752', function()
+      local screen = Screen.new(60, 10)
+      screen:attach()
+      exec_lua([[
+        vim.on_key(function(key, typed)
+          vim.api.nvim_echo({
+            { 'key_cb\n' },
+            { ("KEYCB: key '%s', typed '%s'\n"):format(key, typed) },
+          }, false, {})
+        end)
+      ]])
+      feed('^')
+      screen:expect([[
+                                                                    |
+        {1:~                                                           }|*5
+        {3:                                                            }|
+        key_cb                                                      |
+        KEYCB: key '^', typed '^'                                   |
+        {6:Press ENTER or type command to continue}^                     |
+      ]])
+      feed('<C-C>')
+      screen:expect([[
+                                                                    |
+        {1:~                                                           }|*3
+        {3:                                                            }|
+        key_cb                                                      |
+        KEYCB: key '^', typed '^'                                   |
+        key_cb                                                      |
+        KEYCB: key '{18:^C}', typed '{18:^C}'                                 |
+        {6:Press ENTER or type command to continue}^                     |
+      ]])
+      feed('<C-C>')
+      screen:expect([[
+        ^                                                            |
+        {1:~                                                           }|*8
+                                                                    |
+      ]])
+    end)
   end)
 
   describe('vim.wait', function()