perf(lsp): overscan semantic_token range requests #40036

Problem:
Flickering may occur when paging up/down in big files, as ranges for semantic
tokens are requested. This happens with LSP servers like gopls which return
"/full" semantic tokens if the file is too big, where we fall back to
viewport-range token retrievals.

Solution:
Broaden the requested ranges to one viewport of "overscan" on each side plus
some padding if possible:

    (viewport_topline - viewport_height)..(viewport_botline + viewport_height)

(cherry picked from commit 3ed78daf83)

runtime/lua/vim/lsp/semantic_tokens.lua

@@ -349,7 +349,7 @@ function STHighlighter:send_range_request(client, state, version)
   ---@type lsp.SemanticTokensRangeParams
   local params = {
     textDocument = util.make_text_document_params(self.bufnr),
-    range = self:get_visible_range(),
+    range = self:get_overscan_range(),
   }
 
   ---@type vim.lsp.protocol.Method.ClientToServer.Request
@@ -468,18 +468,21 @@ function STHighlighter:cancel_active_request(client_id)
   clear(state.active_request)
 end
 
---- Gets a range that encompasses all visible lines across all windows
+--- Gets a range that encompasses all visible lines plus overscan across all windows
 --- @private
 --- @return lsp.Range
-function STHighlighter:get_visible_range()
+function STHighlighter:get_overscan_range()
   local wins = vim.fn.win_findbuf(self.bufnr)
+  local num_lines = vim.api.nvim_buf_line_count(self.bufnr)
   local min_start, max_end = nil, nil
 
   for _, win in ipairs(wins) do
     local wininfo = vim.fn.getwininfo(win)[1]
     if wininfo then
-      local start_line = wininfo.topline - 1
-      local end_line = wininfo.botline
+      -- Overscan the token request by the height of the window on each side
+      -- to prevent flickering when navigating up and down.
+      local start_line = math.max(0, wininfo.topline - 1 - wininfo.height)
+      local end_line = math.min(num_lines, wininfo.botline + wininfo.height)
       if not min_start or start_line < min_start then
         min_start = start_line
       end