fix(lsp): limit number of created highlight groups (#39133)

* fix(api): allow silencing "Too many highlight groups" error

Problem: Using Lua's `vim.api.nvim_set_hl(0, 'New', {...})` can fail if
  there are too many existing highlight groups. However, this error can
  not be silenced with `pcall`.

Solution: Make it possible to silence in `nvim_set_hl` and
  `nvim_get_hl_id_by_name`.

* fix(lsp): limit number of groups created by `document_color()`

Problem: A file can contain many string colors that would be highlighted
  by an LSP server. If this number crosses 19999 (maximum number of
  allowed highlight groups), there are general issues with creating
  other highlight groups, which can break functionality outside of
  `vim.lsp.document_color`.

Solution: Limit number of highlight groups that are created by
  `vim.lsp.document_color` to 10000 (half of allowed maximum).
  This is not a 100% solution (since there can exist more than 10000
  other highlight groups), but explicitly checking number of groups is
  slow and 10000 should (hopefully) be enough for most use cases.

runtime/lua/vim/lsp/document_color.lua

@@ -65,12 +65,13 @@ end
 --- Cache of the highlight groups that we've already created.
 --- @type table<string, true>
 local color_cache = {}
+local n_color_cache = 0
 
 --- Gets or creates the highlight group for the given LSP color information.
 ---
 --- @param hex_code string
 --- @param style string
---- @return string
+--- @return string?
 local function get_hl_group(hex_code, style)
   if style ~= 'background' then
     style = 'foreground'
@@ -78,17 +79,23 @@ local function get_hl_group(hex_code, style)
 
   local hl_name = ('LspDocumentColor_%s_%s'):format(hex_code:sub(2), style)
 
-  if not color_cache[hl_name] then
-    if style == 'background' then
-      api.nvim_set_hl(0, hl_name, { bg = hex_code, fg = get_contrast_color(hex_code) })
-    else
-      api.nvim_set_hl(0, hl_name, { fg = hex_code })
-    end
-
-    color_cache[hl_name] = true
+  if color_cache[hl_name] then
+    return hl_name
   end
 
-  return hl_name
+  -- Limit number of created highlight groups to 10000 to not approach near
+  -- a hard limit of 19999 highlight groups (`:h E849`)
+  if n_color_cache > 10000 then
+    return nil
+  end
+
+  local hl_opts = style == 'background' and { bg = hex_code, fg = get_contrast_color(hex_code) }
+    or { fg = hex_code }
+  local ok = pcall(api.nvim_set_hl, 0, hl_name, hl_opts)
+  color_cache[hl_name] = ok
+  n_color_cache = n_color_cache + (ok and 1 or 0)
+
+  return ok and hl_name or nil
 end
 
 --- @class (private) vim.lsp.document_color.Provider : vim.lsp.Capability
@@ -138,6 +145,7 @@ function Provider:new(bufnr)
     desc = 'Refresh document_color',
     callback = function()
       color_cache = {}
+      n_color_cache = 0
       local provider = Provider.active[bufnr]
       if provider then
         provider:clear()

src/nvim/api/vim.c

@@ -92,10 +92,15 @@
 /// Gets a highlight group by name
 ///
 /// similar to |hlID()|, but allocates a new ID if not present.
-Integer nvim_get_hl_id_by_name(String name)
+/// @param[out] err Error details, if any
+Integer nvim_get_hl_id_by_name(String name, Error *err)
   FUNC_API_SINCE(7)
 {
-  return syn_check_group(name.data, name.size);
+  int ret;
+  TRY_WRAP(err, {
+    ret = syn_check_group(name.data, name.size);
+  });
+  return ret;
 }
 
 /// Gets all or specific highlight groups in a namespace.
@@ -173,10 +178,13 @@ DictAs(get_hl_info) nvim_get_hl(Integer ns_id, Dict(get_highlight) *opts, Arena
 void nvim_set_hl(uint64_t channel_id, Integer ns_id, String name, Dict(highlight) *val, Error *err)
   FUNC_API_SINCE(7)
 {
-  int hl_id = syn_check_group(name.data, name.size);
-  VALIDATE_S((hl_id != 0), "highlight name", name.data, {
-    return;
+  int hl_id;
+  TRY_WRAP(err, {
+    hl_id = syn_check_group(name.data, name.size);
   });
+  if (ERROR_SET(err) || hl_id == 0) {
+    return;
+  }
   int link_id = -1;
 
   // Setting URLs directly through highlight attributes is not supported

test/functional/api/highlight_spec.lua

@@ -252,12 +252,25 @@ describe('API: set highlight', function()
 
   it('does not segfault on invalid group name #20009', function()
     eq(
-      "Invalid highlight name: 'foo bar'",
+      'Vim:E5248: Invalid character in group name',
       pcall_err(api.nvim_set_hl, 0, 'foo bar', { bold = true })
     )
     assert_alive()
   end)
 
+  it('can be silenced if there are too many groups #38930', function()
+    local n_groups = vim.tbl_count(api.nvim_get_hl(0, {}))
+    local has_fail = false
+    for i = n_groups + 1, 20000 do
+      local _, msg = pcall(api.nvim_set_hl, 0, 'New_' .. i, { fg = '#000000' })
+      local is_fail = type(msg) == 'string'
+        and msg:find('Too many highlight and syntax groups$') ~= nil
+      has_fail = has_fail or is_fail
+    end
+    eq('', exec_capture('messages'))
+    eq(true, has_fail)
+  end)
+
   it('update=true sets only specified keys', function()
     api.nvim_set_hl(0, 'TestGroup', { fg = '#ff0000', bg = '#0000ff', bold = true })
     api.nvim_set_hl(0, 'TestGroup', { bg = '#00ff00', update = true })