fix(lua): blast radius of broken _G.debug #41507

Problem:
`nlua_pcall()` references `_G.debug.traceback`. If user code deletes it
or breaks it some other way, various Lua features are broken.

    _G.debug = nil
    vim.schedule(function() end)
    vim.wait(100)

    E5113: Lua chunk: attempt to index a nil value
    stack traceback:
            [C]: in function 'loop_poll'
            [string "vim/_core/editor"]:176: in function 'wait'
            crash.lua:3: in main chunk
    PANIC: unprotected error in call to Lua API (attempt to index a nil value)

Solution:
Check `_G.debug.traceback` before using it as errfunc. If it's broken,
omit the traceback and say so in the error message.

Note: We could cache `_G.debug` in LUA_REGISTRYINDEX on startup, but
that would prevent plugins from providing custom functionality there
(and we happen to do so in `tui_spec.lua` for example).

runtime/lua/vim/_core/editor.lua

@@ -769,7 +769,7 @@ function vim._on_key(buf, typed_buf)
     --- @type boolean, any
     local ok, rv = xpcall(function()
       return fn(buf, typed_buf)
-    end, debug.traceback)
+    end, type(debug.traceback) == 'function' and debug.traceback or tostring)
     if ok and rv ~= nil then
       if type(rv) == 'string' and #rv == 0 then
         discard = true

src/nvim/lua/executor.c

@@ -314,21 +314,40 @@ void nlua_error(lua_State *const lstate, const char *const msg)
   lua_pop(lstate, 1);
 }
 
-/// Like lua_pcall, but use debug.traceback as errfunc.
+/// Dummy errfunc used if `_G.debug.traceback` is broken: returns the error object unchanged.
+static int nlua_no_traceback(lua_State *lstate)
+{
+  return 1;
+}
+
+/// Like lua_pcall, but use `debug.traceback` as errfunc. Called by most Lua entrypoints (`:lua`,
+/// `luaeval()`, `v:lua`, API calls, …). Not used by `pcall()` itself.
 ///
 /// @param lstate Lua interpreter state
 /// @param[in] nargs Number of arguments expected by the function being called.
 /// @param[in] nresults Number of results the function returns.
 int nlua_pcall(lua_State *lstate, int nargs, int nresults)
 {
+  // Detect broken `_G.debug` (by user code). #41504
   lua_getglobal(lstate, "debug");
-  lua_getfield(lstate, -1, "traceback");
-  lua_remove(lstate, -2);
+  if (lua_istable(lstate, -1)) {
+    lua_getfield(lstate, -1, "traceback");
+    lua_remove(lstate, -2);
+  }
+  bool no_traceback = !lua_isfunction(lstate, -1);  // `_G.debug.traceback` is a function?
+  if (no_traceback) {
+    lua_pop(lstate, 1);
+    lua_pushcfunction(lstate, nlua_no_traceback);
+  }
   lua_insert(lstate, -2 - nargs);
   int pre_top = lua_gettop(lstate);
   int status = lua_pcall(lstate, nargs, nresults, -2 - nargs);
   if (status) {
     lua_remove(lstate, -2);
+    if (no_traceback && status == LUA_ERRRUN && lua_isstring(lstate, -1)) {
+      lua_pushliteral(lstate, "\nNo traceback: debug.traceback is not a function");
+      lua_concat(lstate, 2);
+    }
   } else {
     if (nresults == LUA_MULTRET) {
       nresults = lua_gettop(lstate) - (pre_top - nargs - 1);

test/functional/lua/overrides_spec.lua

@@ -187,14 +187,56 @@ describe('print', function()
   end)
 end)
 
-describe('debug.debug', function()
+describe('_G.debug', function()
   local screen --- @type test.functional.ui.screen
 
   before_each(function()
     screen = Screen.new()
   end)
 
-  it('works', function()
+  it('if broken, Nvim reports it instead of crashing #41504', function()
+    -- If user code breaks `_G.debug.traceback`, Nvim should function enough to allow ":qa!".
+    -- But `_G.debug=nil` panics immediately, and that is intentional/wontfix.
+    for _, override in ipairs({ 'debug.traceback = nil', '_G.debug = {}' }) do
+      clear()
+      eq(
+        true,
+        exec_lua(([[
+          %s
+          local called = false
+          -- nvim_buf_call() is the shortest path from Lua to nlua_pcall().
+          vim.api.nvim_buf_call(0, function() called = true end)
+          return called
+        ]]):format(override))
+      )
+      -- Error reports the broken situation.
+      matches(
+        'boom\nNo traceback: debug.traceback is not a function',
+        pcall_err(exec_lua, [[vim.api.nvim_buf_call(0, function() error('boom') end)]])
+      )
+      -- vim.on_key() callbacks run on every keystroke, they should not break either.
+      exec_lua([[
+        _G.keys = 0
+        vim.on_key(function() _G.keys = _G.keys + 1 end)
+      ]])
+      feed('jjj')
+      eq(3, exec_lua('return _G.keys'))
+    end
+
+    -- We accept non-broken `debug.traceback` overrides (e.g. tui_spec.lua does this).
+    matches(
+      'boom %(custom traceback%)',
+      pcall_err(
+        exec_lua,
+        [[
+          debug.traceback = function(msg) return msg .. ' (custom traceback)' end
+          vim.api.nvim_buf_call(0, function() error('boom') end)
+        ]]
+      )
+    )
+  end)
+
+  it('debug() works', function()
     command([[lua
       function Test(a)
         print(a)
@@ -262,7 +304,7 @@ describe('debug.debug', function()
     ]])
   end)
 
-  it("can be safely exited with 'cont'", function()
+  it("debug() can be safely exited with 'cont'", function()
     feed('<cr>')
     feed(':lua debug.debug() print("x")<cr>')
     screen:expect {