fix(ui2): ENTER may focus pager in insert/terminal mode #38497

fix(ui2): prevent <CR> from focusing pager in insert/terminal mode

Problem:  <CR> in insert/terminal mode can focus pager unexpectedly.
Solution: Don't enter the pager when <CR> is pressed during expanded
          cmdline in insert/terminal mode.

runtime/doc/lua.txt

@@ -5461,7 +5461,7 @@ and their respective buffers.
 Unlike the legacy |hit-enter| prompt, messages exceeding 'cmdheight' are
 instead "collapsed", followed by a `[+x]` "spill" indicator, where `x`
 indicates the spilled lines. To see the full messages, do either:
-• ENTER immediately after a message from interactive |:| cmdline.
+• ENTER while cmdline is expanded (not in |Insert-mode| and |Terminal-mode|).
 • |g<| at any time.
 
  vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:

runtime/lua/vim/_core/ui2.lua

@@ -43,7 +43,7 @@
 --- Unlike the legacy |hit-enter| prompt, messages exceeding 'cmdheight' are
 --- instead "collapsed", followed by a `[+x]` "spill" indicator, where `x`
 --- indicates the spilled lines. To see the full messages, do either:
---- - ENTER immediately after a message from interactive |:| cmdline.
+--- - ENTER while cmdline is expanded (not in |Insert-mode| and |Terminal-mode|).
 --- - |g<| at any time.
 
 local api = vim.api

runtime/lua/vim/_core/ui2/messages.lua

@@ -517,7 +517,7 @@ local cmd_on_key = function(_, typed)
   M.cmd_on_key, M.cmd.ids = nil, {}
 
   -- Check if window was entered and reopen with original config.
-  local entered = typed == '<CR>'
+  local entered = typed == '<CR>' and not api.nvim_get_mode().mode:match('[it]')
     or typed:find('LeftMouse') and fn.getmousepos().winid == ui.wins.cmd
   pcall(api.nvim_win_close, ui.wins.cmd, true)
   ui.check_targets()

test/functional/ui/messages2_spec.lua

@@ -150,9 +150,8 @@ describe('messages2', function()
       {1:~                                                    }|*11
       foo [+1]                            1,1           All|
     ]])
-    -- Changing 'laststatus' reveals the global statusline with a pager height
+    -- Don't enter the pager in insert mode.
-    -- exceeding the available lines: #38008.
+    command('tabonly | call nvim_echo([["foo\n"]]->repeat(&lines), 1, {}) | startinsert')
-    command('tabonly | call nvim_echo([["foo\n"]]->repeat(&lines), 1, {})')
     screen:expect([[
       ^x                                                    |
       {1:~                                                    }|*5
@@ -161,12 +160,31 @@ describe('messages2', function()
       foo [+8]                                             |
     ]])
     feed('<CR>')
+    screen:expect([[
+                                                           |
+      ^x                                                    |
+      {1:~                                                    }|*11
+      foo [+14]                           2,1           All|
+    ]])
+    feed('<BS><Esc>')
+    command('call nvim_echo([["foo\n"]]->repeat(&lines), 1, {})')
+    screen:expect([[
+      ^x                                                    |
+      {1:~                                                    }|*5
+      {3:                                                     }|
+      foo                                                  |*6
+      foo [+8]                                             |
+    ]])
+    -- Do enter the pager in normal mode.
+    feed('<CR>')
     screen:expect([[
       {3:                                                     }|
       ^foo                                                  |
       foo                                                  |*11
                                           1,1           Top|
     ]])
+    -- Changing 'laststatus' reveals the global statusline with a pager height
+    -- exceeding the available lines: #38008.
     command('set laststatus=3')
     screen:expect([[
       {3:                                                     }|