fix(api): nvim_get_option_value FileType autocmd handling #37414

Problem:
nvim_get_option_value with "filetype" set silently returns incorrect
defaults if autocommands are blocked, like when they're already running.

Solution:
Allow its FileType autocommands to nest: `do_filetype_autocmd(force=true)`.
Also error if executing them fails, rather than silently return wrong defaults.

Endless nesting from misbehaving scripts should be prevented by the recursion
limit in apply_autocmds_group, which is 10.

src/nvim/api/options.c

@@ -136,8 +136,13 @@ static buf_T *do_ft_buf(const char *filetype, aco_save_T *aco, bool *aco_used, E
   ftbuf->b_p_ml = false;
   ftbuf->b_p_ft = xstrdup(filetype);
 
+  if (!has_event(EVENT_FILETYPE)) {
+    return ftbuf;  // Nothing more to do.
+  }
+
+  bool did_au_ft = false;
   TRY_WRAP(err, {
-    do_filetype_autocmd(ftbuf, false);
+    did_au_ft = do_filetype_autocmd(ftbuf, true);
   });
 
   if (!bufref_valid(&bufref)) {
@@ -147,6 +152,9 @@ static buf_T *do_ft_buf(const char *filetype, aco_save_T *aco, bool *aco_used, E
     return NULL;
   }
 
+  if (!did_au_ft && !ERROR_SET(err)) {
+    api_set_error(err, kErrorTypeException, "Could not execute FileType autocommands");
+  }
   return ftbuf;
 }
 

src/nvim/autocmd.c

@@ -2733,12 +2733,13 @@ void do_autocmd_focusgained(bool gained)
   recursive = false;
 }
 
-void do_filetype_autocmd(buf_T *buf, bool force)
+/// @return Whether any FileType autocommands were executed.
+bool do_filetype_autocmd(buf_T *buf, bool force)
 {
   static int ft_recursive = 0;
 
   if (ft_recursive > 0 && !force) {
-    return;  // disallow recursion
+    return false;  // disallow recursion
   }
 
   int secure_save = secure;
@@ -2751,8 +2752,10 @@ void do_filetype_autocmd(buf_T *buf, bool force)
   buf->b_did_filetype = true;
   // Only pass true for "force" when it is true or
   // used recursively, to avoid endless recurrence.
-  apply_autocmds(EVENT_FILETYPE, buf->b_p_ft, buf->b_fname, force || ft_recursive == 1, buf);
+  bool ret
+    = apply_autocmds(EVENT_FILETYPE, buf->b_p_ft, buf->b_fname, force || ft_recursive == 1, buf);
   ft_recursive--;
 
   secure = secure_save;
+  return ret;
 }

test/functional/api/vim_spec.lua

@@ -1969,9 +1969,38 @@ describe('API', function()
         end
       end
 
-      command 'au FileType lua setlocal commentstring=NEW\\ %s'
-
+      command 'au FileType lua ++once setlocal commentstring=NEW\\ %s'
       eq('NEW %s', api.nvim_get_option_value('commentstring', { filetype = 'lua' }))
+
+      -- Works from within a FileType autocommand fired from setting the &filetype.
+      exec [[
+        au FileType * ++once let g:value = nvim_get_option_value('commentstring', #{filetype: 'vim'})
+        set commentstring= ft=lua
+      ]]
+      eq('"%s', eval('g:value'))
+      -- Check it didn't somehow mess up the &commentstring from setting the &filetype.
+      eq('-- %s', eval('&commentstring'))
+
+      -- Not possible to recurse endlessly, of course.
+      exec [[
+        au FileType foobar call nvim_get_option_value('commentstring', #{filetype: 'foobar'})
+      ]]
+      matches( -- Watch out - this error is large!
+        [[E5555: API call: Vim:E218: Autocommand nesting too deep$]],
+        pcall_err(command, 'set ft=foobar')
+      )
+      command('au! FileType foobar')
+
+      eq(
+        [[Vim(call):E5555: API call: Could not execute FileType autocommands]],
+        pcall_err(command, "noautocmd call nvim_get_option_value('tagfunc', #{filetype: 'man'})")
+      )
+
+      -- No error if executed with no FileType autocommands defined.
+      -- Returning the copied global value will continue to suffice, I guess.
+      command([[filetype plugin off | setglobal commentstring=<><\ %s\ ><>]])
+      eq({}, api.nvim_get_autocmds { event = 'FileType' })
+      eq('<>< %s ><>', api.nvim_get_option_value('commentstring', { filetype = 'lua' }))
     end)
 
     it('errors for bad FileType autocmds', function()