Files
neovim/test/functional/plugin/lsp/semantic_tokens_spec.lua
jdrouhard 54188fa242 fix(lsp): make LspNotify more robust #40332
Problem: LspNotify never passed a buffer when executing the autocmds, so
buffer-local LspNotify autocmd subscriptions didn't have the correct buf
in the event metadata. It was also wrapped in a schedule() so the actual
autocmd was delayed until after the event loop.

This could result in the wrong buffer receiving the notification if
multiple LspNotify autocmds with buffer filters were added. Only the
"latest" one would actually receive non-buffer-filtered autocmds, not
the matching one. It also caused listeners to receive the notification
"out of sync" with when the notification is actually sent. If a buffer
is being deleted (which fires a textDocument/didClose notification), the
notification is scheduled and fired after the buffer is already gone.

Solution: For LSP notifications that pertain to a particular buffer, set
it when executing the LspNotify autocmds so the callback functions that
are filtered on that buffer will get the correct notifications and the
metadata buf field will be correct. Additionally, there is no need to
wrap the LspNotify callback in vim.schedule when it can be called inline
when the notification to the rpc server is fired.

This is tested by removing now-unnecessary autocmds from semantic tokens
(InsertEnter and BufWinEnter should no longer be necessary now that
requests are fired by LspNotify). Without this fix, simply modifying a
buffer doesn't actually trigger LspNotify correctly, and the test for
that fails.
2026-06-20 12:46:58 -04:00

2015 lines
67 KiB
Lua

local t = require('test.testutil')
local n = require('test.functional.testnvim')()
local Screen = require('test.functional.ui.screen')
local t_lsp = require('test.functional.plugin.lsp.testutil')
local command = n.command
local dedent = t.dedent
local eq = t.eq
local exec_lua = n.exec_lua
local feed = n.feed
local insert = n.insert
local api = n.api
local clear_notrace = t_lsp.clear_notrace
local create_server_definition = t_lsp.create_server_definition
local function create_start_server()
function _G._start_server(server)
return vim.lsp.start({
name = 'dummy',
cmd = server.cmd,
flags = {
allow_incremental_sync = false,
debounce_text_changes = 0, -- disable debounce to make tests faster
},
})
end
end
before_each(function()
clear_notrace()
exec_lua(create_server_definition)
exec_lua(create_start_server)
exec_lua(function()
vim.lsp.semantic_tokens.__STHighlighter.debounce = 0 -- disable internal debounce to make tests faster
end)
end)
after_each(function()
api.nvim_exec_autocmds('VimLeavePre', { modeline = false })
end)
describe('semantic token highlighting', function()
local screen --- @type test.functional.ui.screen
before_each(function()
screen = Screen.new(40, 16)
screen:set_default_attr_ids {
[1] = { bold = true, foreground = Screen.colors.Blue1 },
[2] = { foreground = Screen.colors.DarkCyan },
[3] = { foreground = Screen.colors.SlateBlue },
[4] = { bold = true, foreground = Screen.colors.SeaGreen },
[5] = { foreground = tonumber('0x6a0dad') },
[6] = { foreground = Screen.colors.Blue1 },
[7] = { bold = true, foreground = Screen.colors.DarkCyan },
[8] = { bold = true, foreground = Screen.colors.SlateBlue },
[9] = { bold = true, foreground = tonumber('0x6a0dad') },
[10] = { bold = true, foreground = Screen.colors.Brown },
[11] = { foreground = Screen.colors.Magenta1 },
}
command([[ hi link @lsp.type.namespace Type ]])
command([[ hi link @lsp.type.function Special ]])
command([[ hi link @lsp.type.comment Comment ]])
command([[ hi @lsp.mod.declaration gui=bold ]])
end)
describe('general', function()
local text = dedent([[
#include <iostream>
int main()
{
int x;
#ifdef __cplusplus
std::cout << x << "\n";
#else
printf("%d\n", x);
#endif
}
}]])
local legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]]
local response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1024, 1, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ],
"resultId": "1"
}]]
local range_response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025 ],
"resultId": "2"
}]]
local edit_response = [[{
"edits": [ {"data": [ 2, 8, 1, 3, 8193, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 3, 8192 ], "deleteCount": 25, "start": 5 } ],
"resultId": "3"
}]]
before_each(function()
exec_lua(function()
_G.server = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
full = { delta = true },
range = false,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(edit_response))
end,
},
})
end)
end)
it('buffer is highlighted when attached', function()
insert(text)
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
vim.bo[bufnr].filetype = 'some-filetype'
_G._start_server(_G.server)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('buffer is highlighted with multiline tokens', function()
insert(text)
exec_lua(function()
_G.server2 = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, {
data = { 5, 0, 82, 0, 0 },
resultId = 1,
})
end,
},
})
end, legend, response, edit_response)
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
vim.bo[bufnr].filetype = 'some-filetype'
_G._start_server(_G.server2)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
{2:#ifdef __cplusplus} |
{2: std::cout << x << "\n";} |
{2:#else} |
{2: printf("%d\n", x);} |
{2:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('calls both range and full when range is supported', function()
insert(text)
exec_lua(function()
_G.server_range = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
range = true,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/range'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(range_response))
end,
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
},
})
end)
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
_G._start_server(_G.server_range)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
local messages = exec_lua('return _G.server_range.messages')
local called_range = false
local called_full = false
for _, m in ipairs(messages) do
if m.method == 'textDocument/semanticTokens/range' then
called_range = true
end
if m.method == 'textDocument/semanticTokens/full' then
called_full = true
end
end
eq(true, called_range)
eq(true, called_full)
end)
it('does not call range when only full is supported', function()
insert(text)
exec_lua(function()
_G.server_full = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
range = false,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
['textDocument/semanticTokens/range'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(range_response))
end,
},
})
return _G._start_server(_G.server_full)
end)
local messages = exec_lua('return _G.server_full.messages')
local called_full = false
local called_range = false
for _, m in ipairs(messages) do
if m.method == 'textDocument/semanticTokens/full' then
called_full = true
end
if m.method == 'textDocument/semanticTokens/range' then
called_range = true
end
end
eq(true, called_full)
eq(false, called_range)
end)
it('does not call range after full request received', function()
insert(text)
exec_lua(function()
_G.server_full = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
full = { delta = false },
range = true,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
['textDocument/semanticTokens/range'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(range_response))
end,
},
})
return _G._start_server(_G.server_full)
end)
-- ensure initial semantic token requests have been sent before feeding input
n.poke_eventloop()
-- modify the buffer
feed('o<ESC>')
local messages = exec_lua('return _G.server_full.messages')
local called_full = 0
local called_range = 0
for _, m in ipairs(messages) do
if m.method == 'textDocument/semanticTokens/full' then
called_full = called_full + 1
end
if m.method == 'textDocument/semanticTokens/range' then
called_range = called_range + 1
end
end
eq(2, called_full)
eq(1, called_range)
end)
it('range requests preserve highlights outside updated range', function()
screen:try_resize(40, 6)
insert(text)
feed('gg')
local client_id, bufnr = exec_lua(function()
_G.response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025 ]
}]]
_G.server2 = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
range = true,
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/range'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(_G.response))
end,
},
})
local bufnr = vim.api.nvim_get_current_buf()
local client_id = assert(_G._start_server(_G.server2))
return client_id, bufnr
end)
screen:expect {
grid = [[
^#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
|
]],
}
eq(
2,
exec_lua(function()
return #vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
exec_lua(function()
_G.response = [[{
"data": [ 7, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ]
}]]
end)
feed('G')
screen:expect {
grid = [[
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
|
]],
}
eq(
5,
exec_lua(function()
return #vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
exec_lua(function()
_G.response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192 ]
}]]
end)
feed('ggLj0')
screen:expect {
grid = [[
|
int {8:main}() |
{ |
int {7:x}; |
^#ifdef {5:__cplusplus} |
|
]],
}
eq(
6,
exec_lua(function()
return #vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
eq(
{
{
line = 2,
end_line = 2,
start_col = 4,
end_col = 8,
marked = true,
modifiers = {
declaration = true,
globalScope = true,
},
type = 'function',
},
{
line = 4,
end_line = 4,
start_col = 8,
end_col = 9,
marked = true,
modifiers = {
declaration = true,
functionScope = true,
},
type = 'variable',
},
{
line = 5,
end_line = 5,
start_col = 7,
end_col = 18,
marked = true,
modifiers = {
globalScope = true,
},
type = 'macro',
},
{
line = 7,
end_line = 7,
start_col = 0,
end_col = 5,
marked = true,
modifiers = {},
type = 'comment',
},
{
line = 8,
end_line = 8,
start_col = 0,
end_col = 22,
marked = true,
modifiers = {},
type = 'comment',
},
{
line = 9,
end_line = 9,
start_col = 0,
end_col = 6,
marked = true,
modifiers = {},
type = 'comment',
},
},
exec_lua(function()
return vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
end)
it('use LspTokenUpdate and highlight_token', function()
insert(text)
exec_lua(function()
vim.api.nvim_create_autocmd('LspTokenUpdate', {
callback = function(ev)
local token = ev.data.token --- @type STTokenRange
if token.type == 'function' and token.modifiers.declaration then
vim.lsp.semantic_tokens.highlight_token(token, ev.buf, ev.data.client_id, 'Macro')
end
end,
})
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
_G._start_server(_G.server)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {9:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('buffer is unhighlighted when client is detached', function()
insert(text)
local bufnr = n.api.nvim_get_current_buf()
local client_id = exec_lua(function()
return _G._start_server(_G.server)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
--- @diagnostic disable-next-line:duplicate-set-field
vim.notify = function() end
vim.lsp.buf_detach_client(bufnr, client_id)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it(
'buffer is highlighted and unhighlighted when semantic token highlighting is enabled and disabled',
function()
insert(text)
exec_lua(function()
return _G._start_server(_G.server)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
--- @diagnostic disable-next-line:duplicate-set-field
vim.notify = function() end
vim.lsp.semantic_tokens.enable(false)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
vim.lsp.semantic_tokens.enable(true)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end
)
it('highlights start and stop when using "0" for current buffer', function()
insert(text)
exec_lua(function()
return _G._start_server(_G.server)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
--- @diagnostic disable-next-line:duplicate-set-field
vim.notify = function() end
vim.lsp.semantic_tokens.enable(false, { bufnr = 0 })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
vim.lsp.semantic_tokens.enable(true, { bufnr = 0 })
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('buffer is re-highlighted when force refreshed', function()
insert(text)
exec_lua(function()
return _G._start_server(_G.server)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
exec_lua(function()
vim.lsp.semantic_tokens.force_refresh()
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
unchanged = true,
}
local messages = exec_lua('return server.messages')
local token_request_count = 0
for _, message in
ipairs(messages --[[@as {method:string,params:table}[] ]])
do
assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received')
if message.method == 'textDocument/semanticTokens/full' then
token_request_count = token_request_count + 1
end
end
eq(2, token_request_count)
end)
it('destroys the highlighter if the buffer is deleted', function()
insert(text)
exec_lua(function()
return _G._start_server(_G.server)
end)
eq(
{},
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_buf_delete(bufnr, { force = true })
return vim.lsp.semantic_tokens.__STHighlighter.active
end)
)
end)
it('updates highlights with delta request on buffer change', function()
insert(text)
exec_lua(function()
return _G._start_server(_G.server)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
feed(':%s/int x/int x()/<CR>')
feed(':noh<CR>')
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
^int {8:x}(); |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {3:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |*2
{1:~ }|*3
:noh |
]],
}
end)
it(
'opt-out: does not activate semantic token highlighting if disabled in client attach',
function()
local client_id = exec_lua(function()
return vim.lsp.start({
name = 'dummy',
cmd = _G.server.cmd,
--- @param client vim.lsp.Client
on_attach = vim.schedule_wrap(function(client, _bufnr)
client.server_capabilities.semanticTokensProvider = nil
end),
})
end)
eq(true, exec_lua('return vim.lsp.buf_is_attached(0, ...)', client_id))
insert(text)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
unchanged = true,
}
end
)
it('ignores null responses from the server', function()
local client_id = exec_lua(function()
_G.server2 = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
full = { delta = false },
},
},
handlers = {
--- @param callback function
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, nil)
end,
--- @param callback function
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(nil, nil)
end,
},
})
return _G._start_server(_G.server2)
end)
eq(
true,
exec_lua(function()
return vim.lsp.buf_is_attached(0, client_id)
end)
)
insert(text)
screen:expect {
grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|*3
|
]],
}
end)
it('resets active request after receiving error responses from the server', function()
local error = { code = -32801, message = 'Content modified' }
exec_lua(function()
_G.server2 = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
full = { delta = false },
},
},
handlers = {
-- There is same logic for handling nil responses and error responses,
-- so keep responses not nil.
--
-- if an error response was not be handled, this test will hang on here.
--- @param callback function
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(error, vim.fn.json_decode(response))
end,
--- @param callback function
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(error, vim.fn.json_decode(response))
end,
},
})
return _G._start_server(_G.server2)
end)
screen:expect([[
^ |
{1:~ }|*14
|
]])
end)
it('does not send delta requests if not supported by server', function()
insert(text)
exec_lua(function()
_G.server2 = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(response))
end,
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(edit_response))
end,
},
})
return _G._start_server(_G.server2)
end)
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
|
]],
}
feed(':%s/int x/int x()/<CR>')
feed(':noh<CR>')
-- the highlights don't change because our fake server sent the exact
-- same result for the same method (the full request). "x" would have
-- changed to highlight index 3 had we sent a delta request
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
^int {7:x}(); |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |*2
{1:~ }|*3
:noh |
]],
}
local messages = exec_lua('return server2.messages')
local token_request_count = 0
for _, message in
ipairs(messages --[[@as {method:string,params:table}[] ]])
do
assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received')
if message.method == 'textDocument/semanticTokens/full' then
token_request_count = token_request_count + 1
end
end
eq(2, token_request_count)
end)
end)
describe('token array decoding', function()
for _, test in ipairs({
{
it = 'clangd-15 on C',
text = [[char* foo = "\n";]],
response = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
expected = {
{
line = 0,
end_line = 0,
modifiers = { declaration = true, globalScope = true },
start_col = 6,
end_col = 9,
type = 'variable',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
char* {7:foo} = "\n"^; |
{1:~ }|*14
|
]],
}
end,
},
{
it = 'clangd-15 on C (bad response data)',
text = [[char* foo = "\n";]],
response = [[{"data": [0, 6, 4294967295, 0, 8193], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
expected = {
{
line = 0,
end_line = 0,
modifiers = { declaration = true, globalScope = true },
start_col = 6,
end_col = 17,
type = 'variable',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
char* {7:foo = "\n"^;} |
{1:~ }|*14
|
]],
}
end,
},
{
it = 'clangd-15 on C++',
text = [[#include <iostream>
int main()
{
#ifdef __cplusplus
const int x = 1;
std::cout << x << std::endl;
#else
comment
#endif
}]],
response = [[{"data": [1, 4, 4, 3, 8193, 2, 9, 11, 19, 8192, 1, 12, 1, 1, 1033, 1, 2, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1032, 0, 5, 3, 15, 8448, 0, 5, 4, 3, 8448, 1, 0, 7, 20, 0, 1, 0, 11, 20, 0, 1, 0, 8, 20, 0], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
expected = {
{ -- main
line = 1,
end_line = 1,
modifiers = { declaration = true, globalScope = true },
start_col = 4,
end_col = 8,
type = 'function',
marked = true,
},
{ -- __cplusplus
line = 3,
end_line = 3,
modifiers = { globalScope = true },
start_col = 9,
end_col = 20,
type = 'macro',
marked = true,
},
{ -- x
line = 4,
end_line = 4,
modifiers = { declaration = true, readonly = true, functionScope = true },
start_col = 12,
end_col = 13,
type = 'variable',
marked = true,
},
{ -- std
line = 5,
end_line = 5,
modifiers = { defaultLibrary = true, globalScope = true },
start_col = 2,
end_col = 5,
type = 'namespace',
marked = true,
},
{ -- cout
line = 5,
end_line = 5,
modifiers = { defaultLibrary = true, globalScope = true },
start_col = 7,
end_col = 11,
type = 'variable',
marked = true,
},
{ -- x
line = 5,
end_line = 5,
modifiers = { readonly = true, functionScope = true },
start_col = 15,
end_col = 16,
type = 'variable',
marked = true,
},
{ -- std
line = 5,
end_line = 5,
modifiers = { defaultLibrary = true, globalScope = true },
start_col = 20,
end_col = 23,
type = 'namespace',
marked = true,
},
{ -- endl
line = 5,
end_line = 5,
modifiers = { defaultLibrary = true, globalScope = true },
start_col = 25,
end_col = 29,
type = 'function',
marked = true,
},
{ -- #else comment #endif
line = 6,
end_line = 6,
modifiers = {},
start_col = 0,
end_col = 7,
type = 'comment',
marked = true,
},
{
line = 7,
end_line = 7,
modifiers = {},
start_col = 0,
end_col = 11,
type = 'comment',
marked = true,
},
{
line = 8,
end_line = 8,
modifiers = {},
start_col = 0,
end_col = 8,
type = 'comment',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
#include <iostream> |
int {8:main}() |
{ |
#ifdef {5:__cplusplus} |
const int {7:x} = 1; |
{4:std}::{2:cout} << {2:x} << {4:std}::{3:endl}; |
{6: #else} |
{6: comment} |
{6: #endif} |
^} |
{1:~ }|*5
|
]],
}
end,
},
{
it = 'sumneko_lua',
text = [[-- comment
local a = 1
b = "as"]],
response = [[{"data": [0, 0, 10, 17, 0, 1, 6, 1, 8, 1, 1, 0, 1, 8, 8]}]],
legend = [[{
"tokenTypes": [
"namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "variable", "property", "enumMember", "event", "function", "method", "macro", "keyword", "modifier", "comment", "string", "number", "regexp", "operator"
],
"tokenModifiers": [
"declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary"
]
}]],
expected = {
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 0,
end_col = 10,
type = 'comment', -- comment
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = { declaration = true }, -- a
start_col = 6,
end_col = 7,
type = 'variable',
marked = true,
},
{
line = 2,
end_line = 2,
modifiers = { static = true }, -- b (global)
start_col = 0,
end_col = 1,
type = 'variable',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
{6:-- comment} |
local {7:a} = 1 |
{2:b} = "as^" |
{1:~ }|*12
|
]],
}
end,
},
{
it = 'rust-analyzer',
text = [[pub fn main() {
println!("Hello world!");
break rust;
/// what?
}
]],
response = [[{"data": [0, 0, 3, 1, 0, 0, 4, 2, 1, 0, 0, 3, 4, 14, 524290, 0, 4, 1, 45, 0, 0, 1, 1, 45, 0, 0, 2, 1, 26, 0, 1, 4, 8, 17, 0, 0, 8, 1, 45, 0, 0, 1, 14, 2, 0, 0, 14, 1, 45, 0, 0, 1, 1, 48, 0, 1, 4, 5, 1, 8192, 0, 6, 4, 52, 0, 0, 4, 1, 48, 0, 1, 4, 9, 0, 1, 1, 0, 1, 26, 0 ], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"comment", "keyword", "string", "number", "regexp", "operator", "namespace", "type", "struct", "class", "interface", "enum", "enumMember", "typeParameter", "function", "method", "property", "macro", "variable",
"parameter", "angle", "arithmetic", "attribute", "attributeBracket", "bitwise", "boolean", "brace", "bracket", "builtinAttribute", "builtinType", "character", "colon", "comma", "comparison", "constParameter", "derive",
"dot", "escapeSequence", "formatSpecifier", "generic", "label", "lifetime", "logical", "macroBang", "operator", "parenthesis", "punctuation", "selfKeyword", "semicolon", "typeAlias", "toolModule", "union", "unresolvedReference"
],
"tokenModifiers": [
"documentation", "declaration", "definition", "static", "abstract", "deprecated", "readonly", "defaultLibrary", "async", "attribute", "callable", "constant", "consuming", "controlFlow", "crateRoot", "injected", "intraDocLink",
"library", "mutable", "public", "reference", "trait", "unsafe"
]
}]],
expected = {
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 0,
end_col = 3, -- pub
type = 'keyword',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 4,
end_col = 6, -- fn
type = 'keyword',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = { declaration = true, public = true },
start_col = 7,
end_col = 11, -- main
type = 'function',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 11,
end_col = 12,
type = 'parenthesis',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 12,
end_col = 13,
type = 'parenthesis',
marked = true,
},
{
line = 0,
end_line = 0,
modifiers = {},
start_col = 14,
end_col = 15,
type = 'brace',
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 4,
end_col = 12,
type = 'macro', -- println!
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 12,
end_col = 13,
type = 'parenthesis',
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 13,
end_col = 27,
type = 'string', -- "Hello world!"
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 27,
end_col = 28,
type = 'parenthesis',
marked = true,
},
{
line = 1,
end_line = 1,
modifiers = {},
start_col = 28,
end_col = 29,
type = 'semicolon',
marked = true,
},
{
line = 2,
end_line = 2,
modifiers = { controlFlow = true },
start_col = 4,
end_col = 9, -- break
type = 'keyword',
marked = true,
},
{
line = 2,
end_line = 2,
modifiers = {},
start_col = 10,
end_col = 14, -- rust
type = 'unresolvedReference',
marked = true,
},
{
line = 2,
end_line = 2,
modifiers = {},
start_col = 14,
end_col = 15,
type = 'semicolon',
marked = true,
},
{
line = 3,
end_line = 3,
modifiers = { documentation = true },
start_col = 4,
end_col = 13,
type = 'comment', -- /// what?
marked = true,
},
{
line = 4,
end_line = 4,
modifiers = {},
start_col = 0,
end_col = 1,
type = 'brace',
marked = true,
},
},
expected_screen = function()
screen:expect {
grid = [[
{10:pub} {10:fn} {8:main}() { |
{5:println!}({11:"Hello world!"}); |
{10:break} rust; |
{6:/// what?} |
} |
^ |
{1:~ }|*9
|
]],
}
end,
},
}) do
it(test.it, function()
local client_id = exec_lua(function(legend, resp)
_G.server = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(resp))
end,
},
})
return _G._start_server(_G.server)
end, test.legend, test.response)
insert(test.text)
test.expected_screen()
eq(
test.expected,
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
return vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
end)
end
end)
describe('token decoding with deltas', function()
for _, test in ipairs({
{
it = 'semantic_tokens_delta: clangd-15 on C',
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
text1 = [[char* foo = "\n";]],
edit = [[ggO<Esc>]],
response1 = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]],
response2 = [[{"edits": [{ "start": 0, "deleteCount": 1, "data": [1] }], "resultId": "2"}]],
expected1 = {
{
line = 0,
modifiers = {
declaration = true,
globalScope = true,
},
start_col = 6,
end_line = 0,
end_col = 9,
type = 'variable',
marked = true,
},
},
expected2 = {
{
line = 1,
modifiers = {
declaration = true,
globalScope = true,
},
start_col = 6,
end_line = 1,
end_col = 9,
type = 'variable',
marked = true,
},
},
expected_screen1 = function()
screen:expect {
grid = [[
char* {7:foo} = "\n"^; |
{1:~ }|*14
|
]],
}
end,
expected_screen2 = function()
screen:expect {
grid = [[
^ |
char* {7:foo} = "\n"; |
{1:~ }|*13
|
]],
}
end,
},
{
it = 'response with multiple delta edits',
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
text1 = dedent([[
#include <iostream>
int main()
{
int x;
#ifdef __cplusplus
std::cout << x << "\n";
#else
printf("%d\n", x);
#endif
}]]),
text2 = [[#include <iostream>
int main()
{
int x();
double y;
#ifdef __cplusplus
std::cout << x << "\n";
#else
printf("%d\n", x);
#endif
}]],
response1 = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1024, 1, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ],
"resultId": 1
}]],
response2 = [[{
"edits": [ {"data": [ 2, 8, 1, 3, 8193, 1, 11, 1, 1, 1025 ], "deleteCount": 5, "start": 5}, {"data": [ 0, 8, 1, 3, 8192 ], "deleteCount": 5, "start": 25 } ],
"resultId":"2"
}]],
expected1 = {
{
line = 2,
end_line = 2,
start_col = 4,
end_col = 8,
modifiers = { declaration = true, globalScope = true },
type = 'function',
marked = true,
},
{
line = 4,
end_line = 4,
start_col = 8,
end_col = 9,
modifiers = { declaration = true, functionScope = true },
type = 'variable',
marked = true,
},
{
line = 5,
end_line = 5,
start_col = 7,
end_col = 18,
modifiers = { globalScope = true },
type = 'macro',
marked = true,
},
{
line = 6,
end_line = 6,
start_col = 4,
end_col = 7,
modifiers = { defaultLibrary = true, globalScope = true },
type = 'namespace',
marked = true,
},
{
line = 6,
end_line = 6,
start_col = 9,
end_col = 13,
modifiers = { defaultLibrary = true, globalScope = true },
type = 'variable',
marked = true,
},
{
line = 6,
end_line = 6,
start_col = 17,
end_col = 18,
marked = true,
modifiers = { functionScope = true },
type = 'variable',
},
{
line = 7,
end_line = 7,
start_col = 0,
end_col = 5,
marked = true,
modifiers = {},
type = 'comment',
},
{
line = 8,
end_line = 8,
end_col = 22,
modifiers = {},
start_col = 0,
type = 'comment',
marked = true,
},
{
line = 9,
end_line = 9,
start_col = 0,
end_col = 6,
modifiers = {},
type = 'comment',
marked = true,
},
},
expected2 = {
{
line = 2,
end_line = 2,
start_col = 4,
end_col = 8,
modifiers = { declaration = true, globalScope = true },
type = 'function',
marked = true,
},
{
line = 4,
end_line = 4,
start_col = 8,
end_col = 9,
modifiers = { declaration = true, globalScope = true },
type = 'function',
marked = true,
},
{
line = 5,
end_line = 5,
end_col = 12,
start_col = 11,
modifiers = { declaration = true, functionScope = true },
type = 'variable',
marked = true,
},
{
line = 6,
end_line = 6,
start_col = 7,
end_col = 18,
modifiers = { globalScope = true },
type = 'macro',
marked = true,
},
{
line = 7,
end_line = 7,
start_col = 4,
end_col = 7,
modifiers = { defaultLibrary = true, globalScope = true },
type = 'namespace',
marked = true,
},
{
line = 7,
end_line = 7,
start_col = 9,
end_col = 13,
modifiers = { defaultLibrary = true, globalScope = true },
type = 'variable',
marked = true,
},
{
line = 7,
end_line = 7,
start_col = 17,
end_col = 18,
marked = true,
modifiers = { globalScope = true },
type = 'function',
},
{
line = 8,
end_line = 8,
start_col = 0,
end_col = 5,
marked = true,
modifiers = {},
type = 'comment',
},
{
line = 9,
end_line = 9,
end_col = 22,
modifiers = {},
start_col = 0,
type = 'comment',
marked = true,
},
{
line = 10,
end_line = 10,
start_col = 0,
end_col = 6,
modifiers = {},
type = 'comment',
marked = true,
},
},
expected_screen1 = function()
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {7:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
^} |
{1:~ }|*4
|
]],
}
end,
expected_screen2 = function()
screen:expect {
grid = [[
#include <iostream> |
|
int {8:main}() |
{ |
int {8:x}(); |
double {7:y}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {3:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:^#endif} |
} |
{1:~ }|*3
|
]],
}
end,
},
{
it = 'optional token_edit.data on deletion',
legend = [[{
"tokenTypes": [
"comment", "keyword", "operator", "string", "number", "regexp", "type", "class", "interface", "enum", "enumMember", "typeParameter", "function", "method", "property", "variable", "parameter", "module", "intrinsic", "selfParameter", "clsParameter", "magicFunction", "builtinConstant", "parenthesis", "curlybrace", "bracket", "colon", "semicolon", "arrow"
],
"tokenModifiers": [
"declaration", "static", "abstract", "async", "documentation", "typeHint", "typeHintComment", "readonly", "decorator", "builtin"
]
}]],
text1 = [[string = "test"]],
text2 = [[]],
response1 = [[{"data": [0, 0, 6, 15, 1], "resultId": "1"}]],
response2 = [[{"edits": [{ "start": 0, "deleteCount": 5 }], "resultId": "2"}]],
expected1 = {
{
line = 0,
end_line = 0,
modifiers = {
declaration = true,
},
start_col = 0,
end_col = 6,
type = 'variable',
marked = true,
},
},
expected2 = {},
expected_screen1 = function()
screen:expect {
grid = [[
{7:string} = "test^" |
{1:~ }|*14
|
]],
}
end,
expected_screen2 = function()
screen:expect {
grid = [[
^ |
{1:~ }|*14
|
]],
}
end,
},
}) do
it(test.it, function()
local bufnr = n.api.nvim_get_current_buf()
insert(test.text1)
local client_id = exec_lua(function(legend, resp1, resp2)
_G.server = _G._create_server({
capabilities = {
textDocumentSync = vim.lsp.protocol.TextDocumentSyncKind.Full,
semanticTokensProvider = {
full = { delta = true },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(resp1))
end,
['textDocument/semanticTokens/full/delta'] = function(_, _, callback)
callback(nil, vim.fn.json_decode(resp2))
end,
},
})
return assert(_G._start_server(_G.server))
end, test.legend, test.response1, test.response2)
test.expected_screen1()
eq(
test.expected1,
exec_lua(function()
return vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
if test.edit then
feed(test.edit)
else
exec_lua(function(text)
vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, vim.fn.split(text, '\n'))
end, test.text2)
end
feed('<Ignore>')
test.expected_screen2()
eq(
test.expected2,
exec_lua(function()
return vim.lsp.semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
end)
)
end)
end
end)
end)