feat(lsp): support multiline semantic tokens #34458

runtime/doc/lsp.txt

@@ -2187,6 +2187,7 @@ get_at_pos({bufnr}, {row}, {col})
         fields:
         • line (integer) line number, 0-based
         • start_col (integer) start column, 0-based
+        • end_line (integer) end line number, 0-based
         • end_col (integer) end column, 0-based
         • type (string) token type as string, e.g. "variable"
         • modifiers (table) token modifiers as a set. E.g., { static = true,

runtime/doc/news.txt

@@ -173,6 +173,7 @@ LSP
   https://microsoft.github.io/language-server-protocol/specifications/specification-current/#workspace_dagnostics
 • Incremental selection is now supported via `textDocument/selectionRange`.
   `an` selects outwards and `in` selects inwards.
+• Support for multiline semantic tokens.
 
 LUA
 

runtime/lua/vim/lsp/protocol.lua

@@ -404,8 +404,7 @@ function protocol.make_client_capabilities()
         },
 
         overlappingTokenSupport = true,
-        -- TODO(jdrouhard): Add support for this
+        multilineTokenSupport = true,
-        multilineTokenSupport = false,
         serverCancelSupport = false,
         augmentsSyntaxTokens = true,
       },

runtime/lua/vim/lsp/semantic_tokens.lua

@@ -2,11 +2,13 @@ local api = vim.api
 local bit = require('bit')
 local ms = require('vim.lsp.protocol').Methods
 local util = require('vim.lsp.util')
+local Range = require('vim.treesitter._range')
 local uv = vim.uv
 
 --- @class (private) STTokenRange
 --- @field line integer line number 0-based
 --- @field start_col integer start column 0-based
+--- @field end_line integer end line number 0-based
 --- @field end_col integer end column 0-based
 --- @field type string token type as string
 --- @field modifiers table<string,boolean> token modifiers as a set. E.g., { static = true, readonly = true }
@@ -44,7 +46,7 @@ local STHighlighter = { active = {} }
 local function lower_bound(tokens, line, lo, hi)
   while lo < hi do
     local mid = bit.rshift(lo + hi, 1) -- Equivalent to floor((lo + hi) / 2).
-    if tokens[mid].line < line then
+    if tokens[mid].end_line < line then
       lo = mid + 1
     else
       hi = mid
@@ -102,6 +104,8 @@ local function tokens_to_ranges(data, bufnr, client, request)
   local token_modifiers = legend.tokenModifiers
   local encoding = client.offset_encoding
   local lines = api.nvim_buf_get_lines(bufnr, 0, -1, false)
+  -- For all encodings, \r\n takes up two code points, and \n (or \r) takes up one.
+  local eol_offset = vim.bo.fileformat[bufnr] == 'dos' and 2 or 1
   local ranges = {} ---@type STTokenRange[]
 
   local start = uv.hrtime()
@@ -141,11 +145,23 @@ local function tokens_to_ranges(data, bufnr, client, request)
     if token_type then
       local modifiers = modifiers_from_number(data[i + 4], token_modifiers)
       local end_char = start_char + data[i + 2] --- @type integer LuaLS bug
-      local buf_line = lines and lines[line + 1] or ''
+      local buf_line = lines[line + 1] or ''
+      local end_line = line ---@type integer
       local start_col = vim.str_byteindex(buf_line, encoding, start_char, false)
       local end_col = vim.str_byteindex(buf_line, encoding, end_char, false)
+
+      end_char = end_char - vim.str_utfindex(buf_line, encoding) - eol_offset
+      -- While end_char goes past the given line, extend the token range to the next line
+      while end_char > 0 do
+        end_line = end_line + 1
+        buf_line = lines[end_line + 1] or ''
+        end_col = vim.str_byteindex(buf_line, encoding, end_char, false)
+        end_char = end_char - vim.str_utfindex(buf_line, encoding) - eol_offset
+      end
+
       ranges[#ranges + 1] = {
         line = line,
+        end_line = end_line,
         start_col = start_col,
         end_col = end_col,
         type = token_type,
@@ -398,6 +414,7 @@ end
 local function set_mark(bufnr, ns, token, hl_group, priority)
   vim.api.nvim_buf_set_extmark(bufnr, ns, token.line, token.start_col, {
     hl_group = hl_group,
+    end_line = token.end_line,
     end_col = token.end_col,
     priority = priority,
     strict = false,
@@ -692,6 +709,7 @@ end
 ---        the following fields:
 ---        - line (integer) line number, 0-based
 ---        - start_col (integer) start column, 0-based
+---        - end_line (integer) end line number, 0-based
 ---        - end_col (integer) end column, 0-based
 ---        - type (string) token type as string, e.g. "variable"
 ---        - modifiers (table) token modifiers as a set. E.g., { static = true, readonly = true }
@@ -709,6 +727,8 @@ function M.get_at_pos(bufnr, row, col)
     row, col = cursor[1] - 1, cursor[2]
   end
 
+  local position = { row, col, row, col }
+
   local tokens = {} --- @type STTokenRangeInspect[]
   for client_id, client in pairs(highlighter.client_state) do
     local highlights = client.current_result.highlights
@@ -722,7 +742,9 @@ function M.get_at_pos(bufnr, row, col)
           break
         end
 
-        if token.start_col <= col and token.end_col > col then
+        if
+          Range.contains({ token.line, token.start_col, token.end_line, token.end_col }, position)
+        then
           token.client_id = client_id
           tokens[#tokens + 1] = token
         end

test/functional/plugin/lsp/semantic_tokens_spec.lua

@@ -132,6 +132,53 @@ describe('semantic token highlighting', function()
       }
     end)
 
+    it('buffer is highlighted with multiline tokens', function()
+      insert(text)
+      exec_lua(function()
+        _G.server2 = _G._create_server({
+          capabilities = {
+            semanticTokensProvider = {
+              full = { delta = true },
+              legend = vim.fn.json_decode(legend),
+            },
+          },
+          handlers = {
+            ['textDocument/semanticTokens/full'] = function(_, _, callback)
+              callback(nil, {
+                data = { 5, 0, 82, 0, 0 },
+                resultId = 1,
+              })
+            end,
+          },
+        })
+      end, legend, response, edit_response)
+      exec_lua(function()
+        local bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        vim.bo[bufnr].filetype = 'some-filetype'
+        vim.lsp.start({ name = 'dummy', cmd = _G.server2.cmd })
+      end)
+
+      screen:expect {
+        grid = [[
+        #include <iostream>                     |
+                                                |
+        int main()                              |
+        {                                       |
+            int x;                              |
+        {2:#ifdef __cplusplus}                      |
+        {2:    std::cout << x << "\n";}             |
+        {2:#else}                                   |
+        {2:    printf("%d\n", x);}                  |
+        {2:#endif}                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|*3
+                                                |
+      ]],
+      }
+    end)
+
     it('use LspTokenUpdate and highlight_token', function()
       insert(text)
       exec_lua(function()
@@ -727,6 +774,7 @@ describe('semantic token highlighting', function()
         expected = {
           {
             line = 0,
+            end_line = 0,
             modifiers = { declaration = true, globalScope = true },
             start_col = 6,
             end_col = 9,
@@ -768,6 +816,7 @@ int main()
         expected = {
           { -- main
             line = 1,
+            end_line = 1,
             modifiers = { declaration = true, globalScope = true },
             start_col = 4,
             end_col = 8,
@@ -776,6 +825,7 @@ int main()
           },
           { --  __cplusplus
             line = 3,
+            end_line = 3,
             modifiers = { globalScope = true },
             start_col = 9,
             end_col = 20,
@@ -784,6 +834,7 @@ int main()
           },
           { -- x
             line = 4,
+            end_line = 4,
             modifiers = { declaration = true, readonly = true, functionScope = true },
             start_col = 12,
             end_col = 13,
@@ -792,6 +843,7 @@ int main()
           },
           { -- std
             line = 5,
+            end_line = 5,
             modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 2,
             end_col = 5,
@@ -800,6 +852,7 @@ int main()
           },
           { -- cout
             line = 5,
+            end_line = 5,
             modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 7,
             end_col = 11,
@@ -808,6 +861,7 @@ int main()
           },
           { -- x
             line = 5,
+            end_line = 5,
             modifiers = { readonly = true, functionScope = true },
             start_col = 15,
             end_col = 16,
@@ -816,6 +870,7 @@ int main()
           },
           { -- std
             line = 5,
+            end_line = 5,
             modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 20,
             end_col = 23,
@@ -824,6 +879,7 @@ int main()
           },
           { -- endl
             line = 5,
+            end_line = 5,
             modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 25,
             end_col = 29,
@@ -832,6 +888,7 @@ int main()
           },
           { -- #else comment #endif
             line = 6,
+            end_line = 6,
             modifiers = {},
             start_col = 0,
             end_col = 7,
@@ -840,6 +897,7 @@ int main()
           },
           {
             line = 7,
+            end_line = 7,
             modifiers = {},
             start_col = 0,
             end_col = 11,
@@ -848,6 +906,7 @@ int main()
           },
           {
             line = 8,
+            end_line = 8,
             modifiers = {},
             start_col = 0,
             end_col = 8,
@@ -891,6 +950,7 @@ b = "as"]],
         expected = {
           {
             line = 0,
+            end_line = 0,
             modifiers = {},
             start_col = 0,
             end_col = 10,
@@ -899,6 +959,7 @@ b = "as"]],
           },
           {
             line = 1,
+            end_line = 1,
             modifiers = { declaration = true }, -- a
             start_col = 6,
             end_col = 7,
@@ -907,6 +968,7 @@ b = "as"]],
           },
           {
             line = 2,
+            end_line = 2,
             modifiers = { static = true }, -- b (global)
             start_col = 0,
             end_col = 1,
@@ -950,6 +1012,7 @@ b = "as"]],
         expected = {
           {
             line = 0,
+            end_line = 0,
             modifiers = {},
             start_col = 0,
             end_col = 3, -- pub
@@ -958,6 +1021,7 @@ b = "as"]],
           },
           {
             line = 0,
+            end_line = 0,
             modifiers = {},
             start_col = 4,
             end_col = 6, -- fn
@@ -966,6 +1030,7 @@ b = "as"]],
           },
           {
             line = 0,
+            end_line = 0,
             modifiers = { declaration = true, public = true },
             start_col = 7,
             end_col = 11, -- main
@@ -974,6 +1039,7 @@ b = "as"]],
           },
           {
             line = 0,
+            end_line = 0,
             modifiers = {},
             start_col = 11,
             end_col = 12,
@@ -982,6 +1048,7 @@ b = "as"]],
           },
           {
             line = 0,
+            end_line = 0,
             modifiers = {},
             start_col = 12,
             end_col = 13,
@@ -990,6 +1057,7 @@ b = "as"]],
           },
           {
             line = 0,
+            end_line = 0,
             modifiers = {},
             start_col = 14,
             end_col = 15,
@@ -998,6 +1066,7 @@ b = "as"]],
           },
           {
             line = 1,
+            end_line = 1,
             modifiers = {},
             start_col = 4,
             end_col = 12,
@@ -1006,6 +1075,7 @@ b = "as"]],
           },
           {
             line = 1,
+            end_line = 1,
             modifiers = {},
             start_col = 12,
             end_col = 13,
@@ -1014,6 +1084,7 @@ b = "as"]],
           },
           {
             line = 1,
+            end_line = 1,
             modifiers = {},
             start_col = 13,
             end_col = 27,
@@ -1022,6 +1093,7 @@ b = "as"]],
           },
           {
             line = 1,
+            end_line = 1,
             modifiers = {},
             start_col = 27,
             end_col = 28,
@@ -1030,6 +1102,7 @@ b = "as"]],
           },
           {
             line = 1,
+            end_line = 1,
             modifiers = {},
             start_col = 28,
             end_col = 29,
@@ -1038,6 +1111,7 @@ b = "as"]],
           },
           {
             line = 2,
+            end_line = 2,
             modifiers = { controlFlow = true },
             start_col = 4,
             end_col = 9, -- break
@@ -1046,6 +1120,7 @@ b = "as"]],
           },
           {
             line = 2,
+            end_line = 2,
             modifiers = {},
             start_col = 10,
             end_col = 14, -- rust
@@ -1054,6 +1129,7 @@ b = "as"]],
           },
           {
             line = 2,
+            end_line = 2,
             modifiers = {},
             start_col = 14,
             end_col = 15,
@@ -1062,6 +1138,7 @@ b = "as"]],
           },
           {
             line = 3,
+            end_line = 3,
             modifiers = { documentation = true },
             start_col = 4,
             end_col = 13,
@@ -1070,6 +1147,7 @@ b = "as"]],
           },
           {
             line = 4,
+            end_line = 4,
             modifiers = {},
             start_col = 0,
             end_col = 1,
@@ -1151,6 +1229,7 @@ b = "as"]],
               globalScope = true,
             },
             start_col = 6,
+            end_line = 0,
             end_col = 9,
             type = 'variable',
             marked = true,
@@ -1164,6 +1243,7 @@ b = "as"]],
               globalScope = true,
             },
             start_col = 6,
+            end_line = 1,
             end_col = 9,
             type = 'variable',
             marked = true,
@@ -1234,6 +1314,7 @@ int main()
         expected1 = {
           {
             line = 2,
+            end_line = 2,
             start_col = 4,
             end_col = 8,
             modifiers = { declaration = true, globalScope = true },
@@ -1242,6 +1323,7 @@ int main()
           },
           {
             line = 4,
+            end_line = 4,
             start_col = 8,
             end_col = 9,
             modifiers = { declaration = true, functionScope = true },
@@ -1250,6 +1332,7 @@ int main()
           },
           {
             line = 5,
+            end_line = 5,
             start_col = 7,
             end_col = 18,
             modifiers = { globalScope = true },
@@ -1258,6 +1341,7 @@ int main()
           },
           {
             line = 6,
+            end_line = 6,
             start_col = 4,
             end_col = 7,
             modifiers = { defaultLibrary = true, globalScope = true },
@@ -1266,6 +1350,7 @@ int main()
           },
           {
             line = 6,
+            end_line = 6,
             start_col = 9,
             end_col = 13,
             modifiers = { defaultLibrary = true, globalScope = true },
@@ -1274,6 +1359,7 @@ int main()
           },
           {
             line = 6,
+            end_line = 6,
             start_col = 17,
             end_col = 18,
             marked = true,
@@ -1282,6 +1368,7 @@ int main()
           },
           {
             line = 7,
+            end_line = 7,
             start_col = 0,
             end_col = 5,
             marked = true,
@@ -1290,6 +1377,7 @@ int main()
           },
           {
             line = 8,
+            end_line = 8,
             end_col = 22,
             modifiers = {},
             start_col = 0,
@@ -1298,6 +1386,7 @@ int main()
           },
           {
             line = 9,
+            end_line = 9,
             start_col = 0,
             end_col = 6,
             modifiers = {},
@@ -1308,6 +1397,7 @@ int main()
         expected2 = {
           {
             line = 2,
+            end_line = 2,
             start_col = 4,
             end_col = 8,
             modifiers = { declaration = true, globalScope = true },
@@ -1316,6 +1406,7 @@ int main()
           },
           {
             line = 4,
+            end_line = 4,
             start_col = 8,
             end_col = 9,
             modifiers = { declaration = true, globalScope = true },
@@ -1324,6 +1415,7 @@ int main()
           },
           {
             line = 5,
+            end_line = 5,
             end_col = 12,
             start_col = 11,
             modifiers = { declaration = true, functionScope = true },
@@ -1332,6 +1424,7 @@ int main()
           },
           {
             line = 6,
+            end_line = 6,
             start_col = 7,
             end_col = 18,
             modifiers = { globalScope = true },
@@ -1340,6 +1433,7 @@ int main()
           },
           {
             line = 7,
+            end_line = 7,
             start_col = 4,
             end_col = 7,
             modifiers = { defaultLibrary = true, globalScope = true },
@@ -1348,6 +1442,7 @@ int main()
           },
           {
             line = 7,
+            end_line = 7,
             start_col = 9,
             end_col = 13,
             modifiers = { defaultLibrary = true, globalScope = true },
@@ -1356,6 +1451,7 @@ int main()
           },
           {
             line = 7,
+            end_line = 7,
             start_col = 17,
             end_col = 18,
             marked = true,
@@ -1364,6 +1460,7 @@ int main()
           },
           {
             line = 8,
+            end_line = 8,
             start_col = 0,
             end_col = 5,
             marked = true,
@@ -1372,6 +1469,7 @@ int main()
           },
           {
             line = 9,
+            end_line = 9,
             end_col = 22,
             modifiers = {},
             start_col = 0,
@@ -1380,6 +1478,7 @@ int main()
           },
           {
             line = 10,
+            end_line = 10,
             start_col = 0,
             end_col = 6,
             modifiers = {},
@@ -1444,6 +1543,7 @@ int main()
         expected1 = {
           {
             line = 0,
+            end_line = 0,
             modifiers = {
               declaration = true,
             },