fix(lsp): multiline semantic token processing #40339

Problem: When multiline semantic token support was introduced, the loop
that finds the end line for a particular token didn't sanitize the token
length sent back by the LSP server. If the server returned an overflowed
length (near uint32 max), neovim would burn cpu and loop for an
extremely long time while trying to find the "end line" represented by
the massively large token, causing neovim to seemingly hang.

Solution: Stop looping once the calculated end_line reaches the actual
last line of the buffer.

Fixes #36257

runtime/lua/vim/lsp/semantic_tokens.lua

@@ -143,7 +143,7 @@ local function tokens_to_ranges(data, bufnr, client, request, ranges)
       ---@type integer LuaLS bug, type must be marked explicitly here
       local new_end_char = end_char - vim.str_utfindex(buf_line, encoding) - eol_offset
       -- While end_char goes past the given line, extend the token range to the next line
-      while new_end_char > 0 do
+      while new_end_char > 0 and end_line < #lines - 1 do
         end_char = new_end_char
         end_line = end_line + 1
         buf_line = lines[end_line + 1] or ''

test/functional/plugin/lsp/semantic_tokens_spec.lua

@@ -1142,6 +1142,39 @@ describe('semantic token highlighting', function()
           }
         end,
       },
+      {
+        it = 'clangd-15 on C (bad response data)',
+        text = [[char* foo = "\n";]],
+        response = [[{"data": [0, 6, 4294967295, 0, 8193], "resultId": "1"}]],
+        legend = [[{
+          "tokenTypes": [
+            "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
+          ],
+          "tokenModifiers": [
+            "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
+          ]
+        }]],
+        expected = {
+          {
+            line = 0,
+            end_line = 0,
+            modifiers = { declaration = true, globalScope = true },
+            start_col = 6,
+            end_col = 17,
+            type = 'variable',
+            marked = true,
+          },
+        },
+        expected_screen = function()
+          screen:expect {
+            grid = [[
+            char* {7:foo = "\n"^;}                       |
+            {1:~                                       }|*14
+                                                    |
+          ]],
+          }
+        end,
+      },
       {
         it = 'clangd-15 on C++',
         text = [[#include <iostream>