Files
neovim/test/functional/plugin/lsp/semantic_tokens_spec.lua
zeertzjq 1983c70d10 test(lsp): flaky lsp/semantic_tokens_spec #38063
The wait added in #37853 doesn't seem to do anything as request is sent
immediately on InsertLeave, and the number 4 also seems wrong. Instead,
the actual cause for the flakiness that the feed() (and hence the buffer
change) may arrive before the scheduled initialization of capabilities,
causing there be only only one textDocument/semanticTokens/full request
instead of two.
2026-02-25 13:04:31 -05:00

1912 lines
64 KiB
Lua

local t = require('test.testutil')
local n = require('test.functional.testnvim')()
local Screen = require('test.functional.ui.screen')
local t_lsp = require('test.functional.plugin.lsp.testutil')
local command = n.command
local dedent = t.dedent
local eq = t.eq
local exec_lua = n.exec_lua
local feed = n.feed
local insert = n.insert
local api = n.api
local clear_notrace = t_lsp.clear_notrace
local create_server_definition = t_lsp.create_server_definition
before_each(function()
clear_notrace()
end)
after_each(function()
api.nvim_exec_autocmds('VimLeavePre', { modeline = false })
end)
describe('semantic token highlighting', function()
local screen --- @type test.functional.ui.screen
before_each(function()
screen = Screen.new(40, 16)
screen:set_default_attr_ids {
[1] = { bold = true, foreground = Screen.colors.Blue1 },
[2] = { foreground = Screen.colors.DarkCyan },
[3] = { foreground = Screen.colors.SlateBlue },
[4] = { bold = true, foreground = Screen.colors.SeaGreen },
[5] = { foreground = tonumber('0x6a0dad') },
[6] = { foreground = Screen.colors.Blue1 },
[7] = { bold = true, foreground = Screen.colors.DarkCyan },
[8] = { bold = true, foreground = Screen.colors.SlateBlue },
[9] = { bold = true, foreground = tonumber('0x6a0dad') },
[10] = { bold = true, foreground = Screen.colors.Brown },
[11] = { foreground = Screen.colors.Magenta1 },
}
command([[ hi link @lsp.type.namespace Type ]])
command([[ hi link @lsp.type.function Special ]])
command([[ hi link @lsp.type.comment Comment ]])
command([[ hi @lsp.mod.declaration gui=bold ]])
end)
describe('general', function()
local text = dedent([[
#include <iostream>
int main()
{
int x;
#ifdef __cplusplus
std::cout << x << "\n";
#else
printf("%d\n", x);
#endif
}
}]])
local legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]]
local response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1024, 1, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ],
"resultId": "1"
}]]
local range_response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025 ],
"resultId": "2"
}]]
local edit_response = [[{
"edits": [ {"data": [ 2, 8, 1, 3, 8193, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 3, 8192 ], "deleteCount": 25, "start": 5 } ],
"resultId": "3"
}]]
before_each(function()
exec_lua(create_server_definition)
exec_lua(function()
_G.server = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = true },
range = false,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(edit_response))
end,
},
})
end)
end)
it('buffer is highlighted when attached', function()
insert(text)
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
vim.bo[bufnr].filetype = 'some-filetype'
vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('buffer is highlighted with multiline tokens', function()
insert(text)
exec_lua(function()
_G.server2 = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, {
data = { 5, 0, 82, 0, 0 },
resultId = 1,
})
end,
},
})
end, legend, response, edit_response)
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
vim.bo[bufnr].filetype = 'some-filetype'
vim.lsp.start({ name = 'dummy', cmd = _G.server2.cmd })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
{2:#ifdef __cplusplus} |
{2: std::cout << x << "\n";} |
{2:#else} |
{2: printf("%d\n", x);} |
{2:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('calls both range and full when range is supported', function()
insert(text)
exec_lua(function()
_G.server_range = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
range = true,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/range'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(range_response))
end,
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
},
})
end)
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
vim.lsp.start({ name = 'dummy', cmd = _G.server_range.cmd })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
local messages = exec_lua('return _G.server_range.messages')
local called_range = false
local called_full = false
for _, m in ipairs(messages) do
if m.method == 'textDocument/semanticTokens/range' then
called_range = true
end
if m.method == 'textDocument/semanticTokens/full' then
called_full = true
end
end
eq(true, called_range)
eq(true, called_full)
end)
it('does not call range when only full is supported', function()
exec_lua(create_server_definition)
insert(text)
exec_lua(function()
_G.server_full = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
range = false,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
['textDocument/semanticTokens/range'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(range_response))
end,
},
})
return vim.lsp.start({ name = 'dummy', cmd = _G.server_full.cmd })
end)
local messages = exec_lua('return _G.server_full.messages')
local called_full = false
local called_range = false
for _, m in ipairs(messages) do
if m.method == 'textDocument/semanticTokens/full' then
called_full = true
end
if m.method == 'textDocument/semanticTokens/range' then
called_range = true
end
end
eq(true, called_full)
eq(false, called_range)
end)
it('does not call range after full request received', function()
exec_lua(create_server_definition)
insert(text)
exec_lua(function()
_G.server_full = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
range = true,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
['textDocument/semanticTokens/range'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(range_response))
end,
},
})
return vim.lsp.start({ name = 'dummy', cmd = _G.server_full.cmd })
end)
-- ensure initial semantic token requests have been sent before feeding input
n.poke_eventloop()
-- modify the buffer
feed('o<ESC>')
local messages = exec_lua('return _G.server_full.messages')
local called_full = 0
local called_range = 0
for _, m in ipairs(messages) do
if m.method == 'textDocument/semanticTokens/full' then
called_full = called_full + 1
end
if m.method == 'textDocument/semanticTokens/range' then
called_range = called_range + 1
end
end
eq(2, called_full)
eq(1, called_range)
end)
it('range requests preserve highlights outside updated range', function()
screen:try_resize(40, 6)
insert(text)
feed('gg')
local client_id, bufnr = exec_lua(function()
_G.response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025 ]
}]]
_G.server2 = _G._create_server({
capabilities = {
semanticTokensProvider = {
range = true,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/range'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(_G.response))
end,
},
})
local bufnr = vim.api.nvim_get_current_buf()
local client_id = assert(vim.lsp.start({ name = 'dummy', cmd = _G.server2.cmd }))
vim.schedule(function()
vim.lsp.semantic_tokens._start(bufnr, client_id, 0)
end)
return client_id, bufnr
end)
screen:expect {
grid = [[
^#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
|
]],
}
eq(
2,
exec_lua(function()
return #vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
exec_lua(function()
_G.response = [[{
"data": [ 7, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ]
}]]
end)
feed('G')
screen:expect {
grid = [[
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
|
]],
}
eq(
5,
exec_lua(function()
return #vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
exec_lua(function()
_G.response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192 ]
}]]
end)
feed('ggLj0')
screen:expect {
grid = [[
|
int {8:main}() |
{ |
int {7:x}; |
^#ifdef {5:__cplusplus} |
|
]],
}
eq(
6,
exec_lua(function()
return #vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
eq(
{
{
line = 2,
end_line = 2,
start_col = 4,
end_col = 8,
marked = true,
modifiers = {
declaration = true,
globalScope = true,
},
type = 'function',
},
{
line = 4,
end_line = 4,
start_col = 8,
end_col = 9,
marked = true,
modifiers = {
declaration = true,
functionScope = true,
},
type = 'variable',
},
{
line = 5,
end_line = 5,
start_col = 7,
end_col = 18,
marked = true,
modifiers = {
globalScope = true,
},
type = 'macro',
},
{
line = 7,
end_line = 7,
start_col = 0,
end_col = 5,
marked = true,
modifiers = {},
type = 'comment',
},
{
line = 8,
end_line = 8,
start_col = 0,
end_col = 22,
marked = true,
modifiers = {},
type = 'comment',
},
{
line = 9,
end_line = 9,
start_col = 0,
end_col = 6,
marked = true,
modifiers = {},
type = 'comment',
},
},
exec_lua(function()
return vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
end)
it('use LspTokenUpdate and highlight_token', function()
insert(text)
exec_lua(function()
vim.api.nvim_create_autocmd('LspTokenUpdate', {
callback = function(args)
local token = args.data.token --- @type STTokenRange
if token.type == 'function' and token.modifiers.declaration then
vim.lsp.semantic_tokens.highlight_token(token, args.buf, args.data.client_id, 'Macro')
end
end,
})
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {9:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('buffer is unhighlighted when client is detached', function()
insert(text)
local bufnr = n.api.nvim_get_current_buf()
local client_id = exec_lua(function()
vim.api.nvim_win_set_buf(0, bufnr)
local client_id = vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
vim.wait(1000, function()
return #_G.server.messages > 1
end)
return client_id
end)
exec_lua(function()
--- @diagnostic disable-next-line:duplicate-set-field
vim.notify = function() end
vim.lsp.buf_detach_client(bufnr, client_id)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it(
'buffer is highlighted and unhighlighted when semantic token highlighting is enabled and disabled',
function()
local bufnr = n.api.nvim_get_current_buf()
exec_lua(function()
vim.api.nvim_win_set_buf(0, bufnr)
return vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
end)
insert(text)
exec_lua(function()
--- @diagnostic disable-next-line:duplicate-set-field
vim.notify = function() end
vim.lsp.semantic_tokens.enable(false)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
vim.lsp.semantic_tokens.enable(true)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end
)
it('highlights start and stop when using "0" for current buffer', function()
exec_lua(function()
return vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
end)
insert(text)
exec_lua(function()
--- @diagnostic disable-next-line:duplicate-set-field
vim.notify = function() end
vim.lsp.semantic_tokens.enable(false, { bufnr = 0 })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
vim.lsp.semantic_tokens.enable(true, { bufnr = 0 })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('buffer is re-highlighted when force refreshed', function()
insert(text)
exec_lua(function()
vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
vim.lsp.semantic_tokens.force_refresh()
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
unchanged = true,
}
local messages = exec_lua('return server.messages')
local token_request_count = 0
for _, message in
ipairs(messages --[[@as {method:string,params:table}[] ]])
do
assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received')
if message.method == 'textDocument/semanticTokens/full' then
token_request_count = token_request_count + 1
end
end
eq(2, token_request_count)
end)
it('destroys the highlighter if the buffer is deleted', function()
exec_lua(function()
vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
end)
insert(text)
eq(
{},
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_buf_delete(bufnr, { force = true })
return vim.lsp.semantic_tokens.__STHighlighter.active
end)
)
end)
it('updates highlights with delta request on buffer change', function()
insert(text)
exec_lua(function()
vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
feed(':%s/int x/int x()/<CR>')
feed(':noh<CR>')
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
^int {8:x}(); |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {3:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |*2
{1:~ }|*3
:noh |
]],
}
end)
it(
'opt-out: does not activate semantic token highlighting if disabled in client attach',
function()
local client_id = exec_lua(function()
return vim.lsp.start({
name = 'dummy',
cmd = _G.server.cmd,
--- @param client vim.lsp.Client
on_attach = vim.schedule_wrap(function(client, _bufnr)
client.server_capabilities.semanticTokensProvider = nil
end),
})
end)
eq(true, exec_lua('return vim.lsp.buf_is_attached(0, ...)', client_id))
insert(text)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
unchanged = true,
}
end
)
it('ignores null responses from the server', function()
local client_id = exec_lua(function()
_G.server2 = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
},
},
handlers = {
--- @param callback function
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, nil)
end,
--- @param callback function
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(nil, nil)
end,
},
})
return vim.lsp.start({ name = 'dummy', cmd = _G.server2.cmd })
end)
eq(
true,
exec_lua(function()
return vim.lsp.buf_is_attached(0, client_id)
end)
)
insert(text)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('resets active request after receiving error responses from the server', function()
local error = { code = -32801, message = 'Content modified' }
exec_lua(function()
_G.server2 = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
},
},
handlers = {
-- There is same logic for handling nil responses and error responses,
-- so keep responses not nil.
--
-- if an error response was not be handled, this test will hang on here.
--- @param callback function
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(error, vim.fn.json_decode(response))
end,
--- @param callback function
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(error, vim.fn.json_decode(response))
end,
},
})
return vim.lsp.start({ name = 'dummy', cmd = _G.server2.cmd })
end)
screen:expect([[
^ |
{1:~ }|*14
|
]])
end)
it('does not send delta requests if not supported by server', function()
insert(text)
exec_lua(function()
_G.server2 = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(edit_response))
end,
},
})
return vim.lsp.start({ name = 'dummy', cmd = _G.server2.cmd })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
feed(':%s/int x/int x()/<CR>')
feed(':noh<CR>')
-- the highlights don't change because our fake server sent the exact
-- same result for the same method (the full request). "x" would have
-- changed to highlight index 3 had we sent a delta request
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
^int {7:x}(); |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |*2
{1:~ }|*3
:noh |
]],
}
local messages = exec_lua('return server2.messages')
local token_request_count = 0
for _, message in
ipairs(messages --[[@as {method:string,params:table}[] ]])
do
assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received')
if message.method == 'textDocument/semanticTokens/full' then
token_request_count = token_request_count + 1
end
end
eq(2, token_request_count)
end)
end)
describe('token array decoding', function()
for _, test in ipairs({
{
it = 'clangd-15 on C',
text = [[char* foo = "\n";]],
response = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
expected = {
{
line = 0,
end_line = 0,
modifiers = { declaration = true, globalScope = true },
start_col = 6,
end_col = 9,
type = 'variable',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
char* {7:foo} = "\n"^; |
{1:~ }|*14
|
]],
}
end,
},
{
it = 'clangd-15 on C++',
text = [[#include <iostream>
int main()
{
#ifdef __cplusplus
const int x = 1;
std::cout << x << std::endl;
#else
comment
#endif
}]],
response = [[{"data": [1, 4, 4, 3, 8193, 2, 9, 11, 19, 8192, 1, 12, 1, 1, 1033, 1, 2, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1032, 0, 5, 3, 15, 8448, 0, 5, 4, 3, 8448, 1, 0, 7, 20, 0, 1, 0, 11, 20, 0, 1, 0, 8, 20, 0], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
expected = {
{ -- main
line = 1,
end_line = 1,
modifiers = { declaration = true, globalScope = true },
start_col = 4,
end_col = 8,
type = 'function',
marked = true,
},
{ -- __cplusplus
line = 3,
end_line = 3,
modifiers = { globalScope = true },
start_col = 9,
end_col = 20,
type = 'macro',
marked = true,
},
{ -- x
line = 4,
end_line = 4,
modifiers = { declaration = true, readonly = true, functionScope = true },
start_col = 12,
end_col = 13,
type = 'variable',
marked = true,
},
{ -- std
line = 5,
end_line = 5,
modifiers = { defaultLibrary = true, globalScope = true },
start_col = 2,
end_col = 5,
type = 'namespace',
marked = true,
},
{ -- cout
line = 5,
end_line = 5,
modifiers = { defaultLibrary = true, globalScope = true },
start_col = 7,
end_col = 11,
type = 'variable',
marked = true,
},
{ -- x
line = 5,
end_line = 5,
modifiers = { readonly = true, functionScope = true },
start_col = 15,
end_col = 16,
type = 'variable',
marked = true,
},
{ -- std
line = 5,
end_line = 5,
modifiers = { defaultLibrary = true, globalScope = true },
start_col = 20,
end_col = 23,
type = 'namespace',
marked = true,
},
{ -- endl
line = 5,
end_line = 5,
modifiers = { defaultLibrary = true, globalScope = true },
start_col = 25,
end_col = 29,
type = 'function',
marked = true,
},
{ -- #else comment #endif
line = 6,
end_line = 6,
modifiers = {},
start_col = 0,
end_col = 7,
type = 'comment',
marked = true,
},
{
line = 7,
end_line = 7,
modifiers = {},
start_col = 0,
end_col = 11,
type = 'comment',
marked = true,
},
{
line = 8,
end_line = 8,
modifiers = {},
start_col = 0,
end_col = 8,
type = 'comment',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
#include <iostream> |
int {8:main}() |
{ |
#ifdef {5:__cplusplus} |
const int {7:x} = 1; |
{4:std}::{2:cout} << {2:x} << {4:std}::{3:endl}; |
{6: #else} |
{6: comment} |
{6: #endif} |
^} |
{1:~ }|*5
|
]],
}
end,
},
{
it = 'sumneko_lua',
text = [[-- comment
local a = 1
b = "as"]],
response = [[{"data": [0, 0, 10, 17, 0, 1, 6, 1, 8, 1, 1, 0, 1, 8, 8]}]],
legend = [[{
"tokenTypes": [
"namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "variable", "property", "enumMember", "event", "function", "method", "macro", "keyword", "modifier", "comment", "string", "number", "regexp", "operator"
],
"tokenModifiers": [
"declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary"
]
}]],
expected = {
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 0,
end_col = 10,
type = 'comment', -- comment
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = { declaration = true }, -- a
start_col = 6,
end_col = 7,
type = 'variable',
marked = true,
},
{
line = 2,
end_line = 2,
modifiers = { static = true }, -- b (global)
start_col = 0,
end_col = 1,
type = 'variable',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
{6:-- comment} |
local {7:a} = 1 |
{2:b} = "as^" |
{1:~ }|*12
|
]],
}
end,
},
{
it = 'rust-analyzer',
text = [[pub fn main() {
println!("Hello world!");
break rust;
/// what?
}
]],
response = [[{"data": [0, 0, 3, 1, 0, 0, 4, 2, 1, 0, 0, 3, 4, 14, 524290, 0, 4, 1, 45, 0, 0, 1, 1, 45, 0, 0, 2, 1, 26, 0, 1, 4, 8, 17, 0, 0, 8, 1, 45, 0, 0, 1, 14, 2, 0, 0, 14, 1, 45, 0, 0, 1, 1, 48, 0, 1, 4, 5, 1, 8192, 0, 6, 4, 52, 0, 0, 4, 1, 48, 0, 1, 4, 9, 0, 1, 1, 0, 1, 26, 0 ], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"comment", "keyword", "string", "number", "regexp", "operator", "namespace", "type", "struct", "class", "interface", "enum", "enumMember", "typeParameter", "function", "method", "property", "macro", "variable",
"parameter", "angle", "arithmetic", "attribute", "attributeBracket", "bitwise", "boolean", "brace", "bracket", "builtinAttribute", "builtinType", "character", "colon", "comma", "comparison", "constParameter", "derive",
"dot", "escapeSequence", "formatSpecifier", "generic", "label", "lifetime", "logical", "macroBang", "operator", "parenthesis", "punctuation", "selfKeyword", "semicolon", "typeAlias", "toolModule", "union", "unresolvedReference"
],
"tokenModifiers": [
"documentation", "declaration", "definition", "static", "abstract", "deprecated", "readonly", "defaultLibrary", "async", "attribute", "callable", "constant", "consuming", "controlFlow", "crateRoot", "injected", "intraDocLink",
"library", "mutable", "public", "reference", "trait", "unsafe"
]
}]],
expected = {
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 0,
end_col = 3, -- pub
type = 'keyword',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 4,
end_col = 6, -- fn
type = 'keyword',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = { declaration = true, public = true },
start_col = 7,
end_col = 11, -- main
type = 'function',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 11,
end_col = 12,
type = 'parenthesis',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 12,
end_col = 13,
type = 'parenthesis',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 14,
end_col = 15,
type = 'brace',
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 4,
end_col = 12,
type = 'macro', -- println!
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 12,
end_col = 13,
type = 'parenthesis',
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 13,
end_col = 27,
type = 'string', -- "Hello world!"
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 27,
end_col = 28,
type = 'parenthesis',
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 28,
end_col = 29,
type = 'semicolon',
marked = true,
},
{
line = 2,
end_line = 2,
modifiers = { controlFlow = true },
start_col = 4,
end_col = 9, -- break
type = 'keyword',
marked = true,
},
{
line = 2,
end_line = 2,
modifiers = {},
start_col = 10,
end_col = 14, -- rust
type = 'unresolvedReference',
marked = true,
},
{
line = 2,
end_line = 2,
modifiers = {},
start_col = 14,
end_col = 15,
type = 'semicolon',
marked = true,
},
{
line = 3,
end_line = 3,
modifiers = { documentation = true },
start_col = 4,
end_col = 13,
type = 'comment', -- /// what?
marked = true,
},
{
line = 4,
end_line = 4,
modifiers = {},
start_col = 0,
end_col = 1,
type = 'brace',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
{10:pub} {10:fn} {8:main}() { |
{5:println!}({11:"Hello world!"}); |
{10:break} rust; |
{6:/// what?} |
} |
^ |
{1:~ }|*9
|
]],
}
end,
},
}) do
it(test.it, function()
exec_lua(create_server_definition)
local client_id = exec_lua(function(legend, resp)
_G.server = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(resp))
end,
},
})
return vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd })
end, test.legend, test.response)
insert(test.text)
test.expected_screen()
eq(
test.expected,
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
return vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
end)
end
end)
describe('token decoding with deltas', function()
for _, test in ipairs({
{
it = 'semantic_tokens_delta: clangd-15 on C',
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
text1 = [[char* foo = "\n";]],
edit = [[ggO<Esc>]],
response1 = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]],
response2 = [[{"edits": [{ "start": 0, "deleteCount": 1, "data": [1] }], "resultId": "2"}]],
expected1 = {
{
line = 0,
modifiers = {
declaration = true,
globalScope = true,
},
start_col = 6,
end_line = 0,
end_col = 9,
type = 'variable',
marked = true,
},
},
expected2 = {
{
line = 1,
modifiers = {
declaration = true,
globalScope = true,
},
start_col = 6,
end_line = 1,
end_col = 9,
type = 'variable',
marked = true,
},
},
expected_screen1 = function()
screen:expect {
grid = [[
char* {7:foo} = "\n"^; |
{1:~ }|*14
|
]],
}
end,
expected_screen2 = function()
screen:expect {
grid = [[
^ |
char* {7:foo} = "\n"; |
{1:~ }|*13
|
]],
}
end,
},
{
it = 'response with multiple delta edits',
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
text1 = dedent([[
#include <iostream>
int main()
{
int x;
#ifdef __cplusplus
std::cout << x << "\n";
#else
printf("%d\n", x);
#endif
}]]),
text2 = [[#include <iostream>
int main()
{
int x();
double y;
#ifdef __cplusplus
std::cout << x << "\n";
#else
printf("%d\n", x);
#endif
}]],
response1 = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1024, 1, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ],
"resultId": 1
}]],
response2 = [[{
"edits": [ {"data": [ 2, 8, 1, 3, 8193, 1, 11, 1, 1, 1025 ], "deleteCount": 5, "start": 5}, {"data": [ 0, 8, 1, 3, 8192 ], "deleteCount": 5, "start": 25 } ],
"resultId":"2"
}]],
expected1 = {
{
line = 2,
end_line = 2,
start_col = 4,
end_col = 8,
modifiers = { declaration = true, globalScope = true },
type = 'function',
marked = true,
},
{
line = 4,
end_line = 4,
start_col = 8,
end_col = 9,
modifiers = { declaration = true, functionScope = true },
type = 'variable',
marked = true,
},
{
line = 5,
end_line = 5,
start_col = 7,
end_col = 18,
modifiers = { globalScope = true },
type = 'macro',
marked = true,
},
{
line = 6,
end_line = 6,
start_col = 4,
end_col = 7,
modifiers = { defaultLibrary = true, globalScope = true },
type = 'namespace',
marked = true,
},
{
line = 6,
end_line = 6,
start_col = 9,
end_col = 13,
modifiers = { defaultLibrary = true, globalScope = true },
type = 'variable',
marked = true,
},
{
line = 6,
end_line = 6,
start_col = 17,
end_col = 18,
marked = true,
modifiers = { functionScope = true },
type = 'variable',
},
{
line = 7,
end_line = 7,
start_col = 0,
end_col = 5,
marked = true,
modifiers = {},
type = 'comment',
},
{
line = 8,
end_line = 8,
end_col = 22,
modifiers = {},
start_col = 0,
type = 'comment',
marked = true,
},
{
line = 9,
end_line = 9,
start_col = 0,
end_col = 6,
modifiers = {},
type = 'comment',
marked = true,
},
},
expected2 = {
{
line = 2,
end_line = 2,
start_col = 4,
end_col = 8,
modifiers = { declaration = true, globalScope = true },
type = 'function',
marked = true,
},
{
line = 4,
end_line = 4,
start_col = 8,
end_col = 9,
modifiers = { declaration = true, globalScope = true },
type = 'function',
marked = true,
},
{
line = 5,
end_line = 5,
end_col = 12,
start_col = 11,
modifiers = { declaration = true, functionScope = true },
type = 'variable',
marked = true,
},
{
line = 6,
end_line = 6,
start_col = 7,
end_col = 18,
modifiers = { globalScope = true },
type = 'macro',
marked = true,
},
{
line = 7,
end_line = 7,
start_col = 4,
end_col = 7,
modifiers = { defaultLibrary = true, globalScope = true },
type = 'namespace',
marked = true,
},
{
line = 7,
end_line = 7,
start_col = 9,
end_col = 13,
modifiers = { defaultLibrary = true, globalScope = true },
type = 'variable',
marked = true,
},
{
line = 7,
end_line = 7,
start_col = 17,
end_col = 18,
marked = true,
modifiers = { globalScope = true },
type = 'function',
},
{
line = 8,
end_line = 8,
start_col = 0,
end_col = 5,
marked = true,
modifiers = {},
type = 'comment',
},
{
line = 9,
end_line = 9,
end_col = 22,
modifiers = {},
start_col = 0,
type = 'comment',
marked = true,
},
{
line = 10,
end_line = 10,
start_col = 0,
end_col = 6,
modifiers = {},
type = 'comment',
marked = true,
},
},
expected_screen1 = function()
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
^} |
{1:~ }|*4
|
]],
}
end,
expected_screen2 = function()
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {8:x}(); |
double {7:y}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {3:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:^#endif} |
} |
{1:~ }|*3
|
]],
}
end,
},
{
it = 'optional token_edit.data on deletion',
legend = [[{
"tokenTypes": [
"comment", "keyword", "operator", "string", "number", "regexp", "type", "class", "interface", "enum", "enumMember", "typeParameter", "function", "method", "property", "variable", "parameter", "module", "intrinsic", "selfParameter", "clsParameter", "magicFunction", "builtinConstant", "parenthesis", "curlybrace", "bracket", "colon", "semicolon", "arrow"
],
"tokenModifiers": [
"declaration", "static", "abstract", "async", "documentation", "typeHint", "typeHintComment", "readonly", "decorator", "builtin"
]
}]],
text1 = [[string = "test"]],
text2 = [[]],
response1 = [[{"data": [0, 0, 6, 15, 1], "resultId": "1"}]],
response2 = [[{"edits": [{ "start": 0, "deleteCount": 5 }], "resultId": "2"}]],
expected1 = {
{
line = 0,
end_line = 0,
modifiers = {
declaration = true,
},
start_col = 0,
end_col = 6,
type = 'variable',
marked = true,
},
},
expected2 = {},
expected_screen1 = function()
screen:expect {
grid = [[
{7:string} = "test^" |
{1:~ }|*14
|
]],
}
end,
expected_screen2 = function()
screen:expect {
grid = [[
^ |
{1:~ }|*14
|
]],
}
end,
},
}) do
it(test.it, function()
local bufnr = n.api.nvim_get_current_buf()
insert(test.text1)
exec_lua(create_server_definition)
local client_id = exec_lua(function(legend, resp1, resp2)
_G.server = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = true },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(resp1))
end,
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(resp2))
end,
},
})
local client_id = assert(vim.lsp.start({ name = 'dummy', cmd = _G.server.cmd }))
-- speed up vim.api.nvim_buf_set_lines calls by changing debounce to 10 for these tests
vim.schedule(function()
vim.lsp.semantic_tokens._start(bufnr, client_id, 10)
end)
return client_id
end, test.legend, test.response1, test.response2)
test.expected_screen1()
eq(
test.expected1,
exec_lua(function()
return vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
if test.edit then
feed(test.edit)
else
exec_lua(function(text)
vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, vim.fn.split(text, '\n'))
vim.wait(15) -- wait for debounce
end, test.text2)
end
test.expected_screen2()
eq(
test.expected2,
exec_lua(function()
return vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
end)
end
end)
end)