fix(ui2): only set dialog on_key callback once #37905

Problem:  vim.on_key() called for each message while cmdline is open.
          Cursor is on a seemingly random column when pager is entered.
          Entering the pager while the cmdline is expanded can be more
          convenient than pressing "g<".
          Pager window is unnecessarily clamped to half the shell height.
          Setting 'laststatus' while pager is open does not adjust its
          dimensions.
Solution: Only call vim.on_key() once when dialog window is opened.
          Ensure cursor is at the start of the first message when
          entering the pager.
          Enter the pager window when "<CR>" is pressed while the
          cmdline is expanded.
          Don't clamp the pager window height.
          Set message windows dimensions when 'laststatus' changes.

runtime/lua/vim/_core/ui2.lua

@@ -141,7 +141,7 @@ function M.check_targets()
 end
 
 local function ui_callback(redraw_msg, event, ...)
-  local handler = M.msg[event] or M.cmd[event]
+  local handler = M.msg[event] or M.cmd[event] --[[@as function]]
   M.check_targets()
   handler(...)
   -- Cmdline mode, non-fast message and non-empty showcmd require an immediate redraw.
@@ -226,9 +226,11 @@ function M.enable(opts)
 
   api.nvim_create_autocmd('OptionSet', {
     group = M.augroup,
-    pattern = { 'cmdheight' },
+    pattern = { 'cmdheight', 'laststatus' },
-    callback = function()
+    callback = function(ev)
-      check_cmdheight(vim.v.option_new)
+      if ev.match == 'cmdheight' then
+        check_cmdheight(vim.v.option_new)
+      end
       M.msg.set_pos()
     end,
     desc = 'Set cmdline and message window dimensions for changed option values.',

runtime/lua/vim/_core/ui2/cmdline.lua

@@ -143,7 +143,7 @@ function M.cmdline_hide(level, abort)
       api.nvim_buf_set_lines(ui.bufs.dialog, 0, -1, false, {})
       api.nvim_win_set_config(ui.wins.dialog, { hide = true })
       vim.on_key(nil, ui.msg.dialog_on_key)
-      M.dialog = false
+      M.dialog, ui.msg.dialog_on_key = false, nil
     end
   end)
 

runtime/lua/vim/_core/ui2/messages.lua

@@ -35,7 +35,7 @@ local M = {
     ids = {}, ---@type { ['last'|'msg'|'top'|'bot']: integer? } Table of mark IDs.
     delayed = false, -- Whether placement of 'last' virt_text is delayed.
   },
-  dialog_on_key = 0, -- vim.on_key namespace for paging in the dialog window.
+  dialog_on_key = nil, ---@type integer? vim.on_key namespace for paging in the dialog window.
 }
 
 local cmd_on_key ---@type integer? Set to vim.on_key namespace while cmdline is expanded.
@@ -507,7 +507,8 @@ function M.set_pos(type)
     local cfg = { hide = false, relative = 'laststatus', col = 10000 }
     local texth = type and api.nvim_win_text_height(win, {}) or {}
     local top = { vim.opt.fcs:get().msgsep or ' ', 'MsgSeparator' }
-    cfg.height = type and math.min(texth.all, math.ceil(o.lines * 0.5))
+    cfg.height = type == 'pager' and texth.all
+      or type and math.min(texth.all, math.ceil(o.lines * 0.5))
     cfg.border = win ~= ui.wins.msg and { '', top, '', '', '', '', '', '' } or nil
     cfg.focusable = type == 'cmd' or nil
     cfg.row = (win == ui.wins.msg and 0 or 1) - ui.cmd.wmnumode
@@ -522,11 +523,13 @@ function M.set_pos(type)
       set_virttext('msg', 'cmd')
       M.virt.msg[M.virt.idx.spill][1] = save_spill
       cmd_on_key = vim.on_key(function(_, typed)
-        if not typed or fn.keytrans(typed) == '<MouseMove>' then
+        typed = typed and fn.keytrans(typed)
+        if not typed or typed == '<MouseMove>' then
           return
         end
+
         vim.schedule(function()
-          local entered = api.nvim_get_current_win() == ui.wins.cmd
+          local entered = typed == '<CR>' or api.nvim_get_current_win() == ui.wins.cmd
           cmd_on_key = nil
           if api.nvim_win_is_valid(ui.wins.cmd) then
             api.nvim_win_close(ui.wins.cmd, true)
@@ -537,7 +540,8 @@ function M.set_pos(type)
             M.virt.msg[M.virt.idx.spill][1] = nil
             api.nvim_buf_set_lines(ui.bufs.cmd, 0, -1, false, {})
             if entered then
-              api.nvim_command('norm! g<') -- User entered the cmdline window: open the pager.
+              -- User entered the cmdline window or pressed enter: open the pager.
+              api.nvim_command('norm! g<')
             end
           elseif ui.cfg.msg.target == 'cmd' and ui.cmd.level == 0 then
             ui.check_targets()
@@ -581,7 +585,7 @@ function M.set_pos(type)
           set_top_bot_spill()
           return fn.getwininfo(ui.wins.dialog)[1].topline ~= info.topline and '' or nil
         end
-      end)
+      end, M.dialog_on_key)
     elseif type == 'msg' then
       -- Ensure last line is visible and first line is at top of window.
       local row = (texth.all > cfg.height and texth.end_row or 0) + 1
@@ -597,6 +601,8 @@ function M.set_pos(type)
       -- Cmdwin is actually closed one event iteration later so schedule in case it was open.
       vim.schedule(function()
         api.nvim_set_current_win(ui.wins.pager)
+        -- Ensure cursor is at beginning of first message.
+        api.nvim_win_set_cursor(ui.wins.pager, { 1, 0 })
         -- Make pager relative to cmdwin when it is opened, restore when it is closed.
         api.nvim_create_autocmd({ 'WinEnter', 'CmdwinEnter', 'CmdwinLeave' }, {
           callback = function(ev)

test/functional/ui/messages2_spec.lua

@@ -45,9 +45,9 @@ describe('messages2', function()
                                                            |
       {1:~                                                    }|*9
       {3:                                                     }|
-      fo^o                                                  |
+      ^foo                                                  |
       bar                                                  |
-                                          1,3           All|
+                                          1,1           All|
     ]])
     -- Multiple messages in same event loop iteration are appended and shown in full.
     feed([[q:echo "foo" | echo "bar\nbaz\n"->repeat(&lines)<CR>]])
@@ -100,10 +100,10 @@ describe('messages2', function()
                                                            |
       {1:~                                                    }|*8
       {3:                                                     }|
-      fo^o                                                  |
+      ^foo                                                  |
       bar                                                  |
         1 %a   "[No Name]"                    line 1       |
-                                          1,3           All|
+                                          1,1           All|
     ]])
     -- edit_unputchar() does not clear already updated screen #34515.
     feed('qix<Esc>dwi<C-r>')
@@ -143,7 +143,7 @@ describe('messages2', function()
                                                            |
       {1:~                                                    }|*10
       {3:                                                     }|
-      fo^o                                                  |
+      ^foo                                                  |
       foo                                                  |
     ]])
     command('bdelete | messages')
@@ -417,7 +417,7 @@ describe('messages2', function()
                                                            |
       {1:~                                                    }|*10
       {3:                                                     }|
-      foofoofoofoofoofoofoofoofo^o                          |
+      ^foofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofo|
                                                            |
     ]])
     t.eq({ filetype = 5 }, n.eval('g:set')) -- still fires for 'filetype'
@@ -448,7 +448,7 @@ describe('messages2', function()
                                                            |
       {1:~                                                    }|*11
       {3:                                                     }|
-      {101:fo^o}{100:                                                  }|
+      {101:^foo}{100:                                                  }|
     ]])
   end)
 
@@ -564,7 +564,7 @@ describe('messages2', function()
                                                            |
       {1:~                                                    }|*8
       {3:                                                     }|
-      x^!                                                   |
+      ^x!                                                   |
       x!                                                   |
       i hate locks so much!!!!                             |*2
     ]])
@@ -647,7 +647,7 @@ describe('messages2', function()
       foo                                                  |*2
       {14:f}oo                                                  |
     ]])
-    feed('<CR>')
+    feed('<Esc>')
     screen:expect([[
       ^                                                     |
       {1:~                                                    }|*5