fix(startup): wait for bg detection before user config #37075

Problem:
Automatic background detection sets the background option too late,
which loads colorschemes twice and causes problems when the user's
terminal background doesn't match the default (#32109, #36211, #36416).

Solution:
Use a DA1 query to determine whether the TTY supports OSC 11. Wait for
background detection and setting to complete before processing user
config.

Note: To preserve the existing behavior as much as possible, this triggers
OptionSet manually on VimEnter (since it won't trigger automatically if
we set bg during startup). However, I'm unsure if this behavior is
truly desired given that the documentation says OptionSet is triggered
"After setting an option (except during |startup|)."

Also fixes flickering issue #28667. To check for flickering:

    nvim --clean --cmd "set termguicolors" --cmd "echo \"foo\"" --cmd "sleep 10"

On master, this gives me a black screen for 10 seconds, but on this
branch, the background is dark or light depending on the terminal
background (since the option is now set during startup rather than after
VimEnter).

runtime/lua/vim/_core/defaults.lua

@@ -814,13 +814,25 @@ do
       -- an OSC 11 response from the terminal emulator. If the user has set
       -- 'background' explicitly then we will delete this autocommand,
       -- effectively disabling automatic background setting.
-      local force = false
+      local did_dsr_response = false
       local id = vim.api.nvim_create_autocmd('TermResponse', {
         group = group,
         nested = true,
         desc = "Update the value of 'background' automatically based on the terminal emulator's background color",
         callback = function(args)
           local resp = args.data.sequence ---@type string
+
+          -- DSR response that should come after the OSC 11 response if the
+          -- terminal supports it.
+          if string.match(resp, '^\027%[0n$') then
+            did_dsr_response = true
+            -- Don't delete the autocmd because the bg response may come
+            -- after the DSR response if the terminal handles requests out
+            -- of sequence. In that case, the background will simply be set
+            -- later in the startup sequence.
+            return false
+          end
+
           local r, g, b = parseosc11(resp)
           if r and g and b then
             local rr = parsecolor(r)
@@ -830,15 +842,20 @@ do
             if rr and gg and bb then
               local luminance = (0.299 * rr) + (0.587 * gg) + (0.114 * bb)
               local bg = luminance < 0.5 and 'dark' or 'light'
-              setoption('background', bg, force)
+              vim.api.nvim_set_option_value('background', bg, {})
 
-              -- On the first query response, don't force setting the option in
-              -- case the user has already set it manually. If they have, then
-              -- this autocommand will be deleted. If they haven't, then we do
-              -- want to force setting the option to override the value set by
-              -- this autocommand.
-              if not force then
-                force = true
+              -- Ensure OptionSet still triggers when we set the background during startup
+              if vim.v.vim_did_enter == 0 then
+                vim.api.nvim_create_autocmd('VimEnter', {
+                  group = group,
+                  once = true,
+                  nested = true,
+                  callback = function()
+                    vim.api.nvim_exec_autocmds('OptionSet', {
+                      pattern = 'background',
+                    })
+                  end,
+                })
               end
             end
           end
@@ -850,13 +867,37 @@ do
         nested = true,
         once = true,
         callback = function()
-          if vim.api.nvim_get_option_info2('background', {}).was_set then
+          local optinfo = vim.api.nvim_get_option_info2('background', {})
+          local sid_lua = -8
+          if
+            optinfo.was_set
+            and optinfo.last_set_sid ~= sid_lua
+            and next(vim.api.nvim_get_autocmds({ id = id })) ~= nil
+          then
             vim.api.nvim_del_autocmd(id)
           end
         end,
       })
 
-      vim.api.nvim_ui_send('\027]11;?\007')
+      -- Send OSC 11 query along with DSR sequence to determine whether
+      -- terminal supports the query. If the DSR response comes first,
+      -- the terminal most likely doesn't support the bg color query,
+      -- and we don't have to keep waiting for a bg color response.
+      -- #32109
+      local osc11 = '\027]11;?\007'
+      local dsr = '\027[5n'
+      vim.api.nvim_ui_send(osc11 .. dsr)
+
+      -- Wait until detection of OSC 11 capabilities is complete to
+      -- ensure background is automatically set before user config.
+      if not vim.wait(100, function()
+        return did_dsr_response
+      end, 1) then
+        vim.notify(
+          'defaults.lua: Did not detect DSR response from terminal. This results in a slower startup time.',
+          vim.log.levels.WARN
+        )
+      end
     end
 
     --- If the TUI (term_has_truecolor) was able to determine that the host

src/nvim/tui/input.c

@@ -726,7 +726,31 @@ static void handle_unknown_csi(TermInput *input, const TermKeyKey *key)
     break;
   case 'n':
     // Device Status Report (DSR)
-    if (nparams == 2) {
+    if (nparams == 1) {
+      // ECMA-48 DSR
+      // https://ecma-international.org/wp-content/uploads/ECMA-48_5th_edition_june_1991.pdf
+      int arg;
+      if (termkey_interpret_csi_param(params[0], &arg, NULL, NULL) != TERMKEY_RES_KEY) {
+        return;
+      }
+
+      MAXSIZE_TEMP_ARRAY(args, 2);
+      ADD_C(args, STATIC_CSTR_AS_OBJ("termresponse"));
+
+      StringBuilder response = KV_INITIAL_VALUE;
+      kv_printf(response, "\x1b[%dn", arg);
+      ADD_C(args, STRING_OBJ(cbuf_as_string(response.items, response.size)));
+
+      rpc_send_event(ui_client_channel_id, "nvim_ui_term_event", args);
+      kv_destroy(response);
+    } else if (nparams == 2) {
+      // Hard to find comprehensive docs on these responses. Some can be found at https://www.xfree86.org/current/ctlseqs.html
+      // under "Device Status Report (DSR, DEC-specific)"
+      // - Report Printer status
+      // - Report User Defined Key status
+      // - Report Locator status
+      // When the first parameter is 997, it's a theme update response based on
+      // contour terminal VT extensions, as described below.
       int args[2];
       for (size_t i = 0; i < ARRAY_SIZE(args); i++) {
         if (termkey_interpret_csi_param(params[i], &args[i], NULL, NULL) != TERMKEY_RES_KEY) {

src/nvim/tui/tui.c

@@ -337,13 +337,14 @@ static void tui_reset_key_encoding(TUIData *tui)
   }
 }
 
-/// Write the OSC 11 sequence to the terminal emulator to query the current background color.
+/// Write the OSC 11 + DSR sequence to the terminal emulator to query the current
+/// background color.
 ///
 /// Response will be handled by the TermResponse handler in _core/defaults.lua.
 void tui_query_bg_color(TUIData *tui)
   FUNC_ATTR_NONNULL_ALL
 {
-  out(tui, S_LEN("\x1b]11;?\x07"));
+  out(tui, S_LEN("\x1b]11;?\x07\x1b[5n"));
   flush_buf(tui);
 }
 

test/functional/terminal/tui_spec.lua

@@ -2891,10 +2891,15 @@ describe('TUI', function()
   for _, guicolors in ipairs({ 'notermguicolors', 'termguicolors' }) do
     it('has no black flicker when clearing regions during startup with ' .. guicolors, function()
       local screen = Screen.new(50, 10)
+      -- Colorscheme is automatically detected as light in _core/defaults.lua, so fg
+      -- should be dark except on Windows, where it doesn't respond to the OSC11 query,
+      -- so bg is dark.
+      local fg = is_os('win') and Screen.colors.NvimLightGrey2 or Screen.colors.NvimDarkGrey2
+      local bg = is_os('win') and Screen.colors.NvimDarkGrey2 or Screen.colors.NvimLightGrey2
       screen:add_extra_attr_ids({
         [100] = {
-          foreground = Screen.colors.NvimLightGrey2,
-          background = Screen.colors.NvimDarkGrey2,
+          foreground = fg,
+          background = bg,
         },
       })
       fn.jobstart({
@@ -2908,6 +2913,7 @@ describe('TUI', function()
         'sleep 10',
       }, {
         term = true,
+        env = { VIMRUNTIME = os.getenv('VIMRUNTIME') },
       })
       if guicolors == 'termguicolors' then
         screen:expect([[