Files
neovim/test/functional/plugin/lsp/semantic_tokens_spec.lua
John Drouhard 9f035559de feat(lsp): initial support for semantic token highlighting
* credit to @smolck and @theHamsta for their contributions in laying the
  groundwork for this feature and for their work on some of the helper
  utility functions and tests
2022-12-08 11:31:56 -06:00

911 lines
32 KiB
Lua

local helpers = require('test.functional.helpers')(after_each)
local lsp_helpers = require('test.functional.plugin.lsp.helpers')
local Screen = require('test.functional.ui.screen')
local command = helpers.command
local dedent = helpers.dedent
local eq = helpers.eq
local exec_lua = helpers.exec_lua
local feed = helpers.feed
local feed_command = helpers.feed_command
local insert = helpers.insert
local matches = helpers.matches
local clear_notrace = lsp_helpers.clear_notrace
local create_server_definition = lsp_helpers.create_server_definition
before_each(function()
clear_notrace()
end)
after_each(function()
exec_lua("vim.api.nvim_exec_autocmds('VimLeavePre', { modeline = false })")
end)
describe('semantic token highlighting', function()
describe('general', function()
local text = dedent([[
#include <iostream>
int main()
{
int x;
#ifdef __cplusplus
std::cout << x << "\n";
#else
printf("%d\n", x);
#endif
}
}]])
local legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]]
local response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1024, 1, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ],
"resultId": 1
}]]
local edit_response = [[{
"edits": [ {"data": [ 2, 8, 1, 3, 8193, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 3, 8192 ], "deleteCount": 25, "start": 5 } ],
"resultId":"2"
}]]
local screen
before_each(function()
screen = Screen.new(40, 16)
screen:attach()
screen:set_default_attr_ids {
[1] = { bold = true, foreground = Screen.colors.Blue1 };
[2] = { foreground = Screen.colors.DarkCyan };
[3] = { foreground = Screen.colors.SlateBlue };
[4] = { bold = true, foreground = Screen.colors.SeaGreen };
[5] = { foreground = tonumber('0x6a0dad') };
[6] = { foreground = Screen.colors.Blue1 };
}
command([[ hi link @namespace Type ]])
command([[ hi link @function Special ]])
exec_lua(create_server_definition)
exec_lua([[
local legend, response, edit_response = ...
server = _create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = true },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function()
return vim.fn.json_decode(response)
end,
['textDocument/semanticTokens/full/delta'] = function()
return vim.fn.json_decode(edit_response)
end,
}
})
]], legend, response, edit_response)
end)
it('buffer is highlighted when attached', function()
exec_lua([[
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
]])
insert(text)
screen:expect { grid = [[
#include <iostream> |
|
int {3:main}() |
{ |
int {2:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]] }
end)
it('buffer is unhighlighted when client is detached', function()
exec_lua([[
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
]])
insert(text)
exec_lua([[
vim.notify = function() end
vim.lsp.buf_detach_client(bufnr, client_id)
]])
screen:expect { grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]] }
end)
it('buffer is highlighted and unhighlighted when semantic token highlighting is started and stopped'
, function()
exec_lua([[
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
]])
insert(text)
exec_lua([[
vim.notify = function() end
vim.lsp.semantic_tokens.stop(bufnr, client_id)
]])
screen:expect { grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]] }
exec_lua([[
vim.lsp.semantic_tokens.start(bufnr, client_id)
]])
screen:expect { grid = [[
#include <iostream> |
|
int {3:main}() |
{ |
int {2:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]] }
end)
it('buffer is re-highlighted when force refreshed', function()
exec_lua([[
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
]])
insert(text)
screen:expect { grid = [[
#include <iostream> |
|
int {3:main}() |
{ |
int {2:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]] }
exec_lua([[
vim.lsp.semantic_tokens.force_refresh(bufnr)
]])
screen:expect { grid = [[
#include <iostream> |
|
int {3:main}() |
{ |
int {2:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]], unchanged = true }
local messages = exec_lua('return server.messages')
local token_request_count = 0
for _, message in ipairs(messages) do
assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received')
if message.method == 'textDocument/semanticTokens/full' then
token_request_count = token_request_count + 1
end
end
eq(2, token_request_count)
end)
it('destroys the highlighter if the buffer is deleted', function()
exec_lua([[
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
]])
insert(text)
local highlighters = exec_lua([[
vim.api.nvim_buf_delete(bufnr, { force = true })
local semantic_tokens = vim.lsp.semantic_tokens
return semantic_tokens.__STHighlighter.active
]])
eq({}, highlighters)
end)
it('updates highlights with delta request on buffer change', function()
exec_lua([[
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
]])
insert(text)
feed_command('%s/int x/int x()/')
feed_command('noh')
screen:expect { grid = [[
#include <iostream> |
|
int {3:main}() |
{ |
^int {3:x}(); |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {3:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
} |
{1:~ }|
{1:~ }|
{1:~ }|
:noh |
]] }
end)
it('prevents starting semantic token highlighting with invalid conditions', function()
exec_lua([[
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start_client({ name = 'dummy', cmd = server.cmd })
notifications = {}
vim.notify = function(...) table.insert(notifications, 1, {...}) end
]])
eq(false, exec_lua("return vim.lsp.buf_is_attached(bufnr, client_id)"))
insert(text)
local notifications = exec_lua([[
vim.lsp.semantic_tokens.start(bufnr, client_id)
return notifications
]])
matches('%[LSP%] Client with id %d not attached to buffer %d', notifications[1][1])
notifications = exec_lua([[
vim.lsp.semantic_tokens.start(bufnr, client_id + 1)
return notifications
]])
matches('%[LSP%] No client with id %d', notifications[1][1])
end)
it('opt-out: does not activate semantic token highlighting if disabled in client attach',
function()
exec_lua([[
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({
name = 'dummy',
cmd = server.cmd,
on_attach = function(client, bufnr)
client.server_capabilities.semanticTokensProvider = nil
end,
})
]])
eq(true, exec_lua("return vim.lsp.buf_is_attached(bufnr, client_id)"))
insert(text)
screen:expect { grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]] }
local notifications = exec_lua([[
local notifications = {}
vim.notify = function(...) table.insert(notifications, 1, {...}) end
vim.lsp.semantic_tokens.start(bufnr, client_id)
return notifications
]])
eq('[LSP] Server does not support semantic tokens', notifications[1][1])
screen:expect { grid = [[
#include <iostream> |
|
int main() |
{ |
int x; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]], unchanged = true }
end)
it('does not send delta requests if not supported by server', function()
exec_lua([[
local legend, response, edit_response = ...
server2 = _create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function()
return vim.fn.json_decode(response)
end,
['textDocument/semanticTokens/full/delta'] = function()
return vim.fn.json_decode(edit_response)
end,
}
})
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server2.cmd })
]], legend, response, edit_response)
insert(text)
screen:expect { grid = [[
#include <iostream> |
|
int {3:main}() |
{ |
int {2:x}; |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|
{1:~ }|
{1:~ }|
|
]] }
feed_command('%s/int x/int x()/')
feed_command('noh')
-- the highlights don't change because our fake server sent the exact
-- same result for the same method (the full request). "x" would have
-- changed to highlight index 3 had we sent a delta request
screen:expect { grid = [[
#include <iostream> |
|
int {3:main}() |
{ |
^int {2:x}(); |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
} |
{1:~ }|
{1:~ }|
{1:~ }|
:noh |
]] }
local messages = exec_lua('return server2.messages')
local token_request_count = 0
for _, message in ipairs(messages) do
assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received')
if message.method == 'textDocument/semanticTokens/full' then
token_request_count = token_request_count + 1
end
end
eq(2, token_request_count)
end)
end)
describe('token array decoding', function()
for _, test in ipairs({
{
it = 'clangd-15 on C',
text = [[char* foo = "\n";]],
response = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
expected = {
{
line = 0,
modifiers = {
'declaration',
'globalScope',
},
start_col = 6,
end_col = 9,
type = 'variable',
extmark_added = true,
},
},
},
{
it = 'clangd-15 on C++',
text = [[#include <iostream>
int main()
{
#ifdef __cplusplus
const int x = 1;
std::cout << x << std::endl;
#else
comment
#endif
}]] ,
response = [[{"data": [1, 4, 4, 3, 8193, 2, 9, 11, 19, 8192, 1, 12, 1, 1, 1033, 1, 2, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1032, 0, 5, 3, 15, 8448, 0, 5, 4, 3, 8448, 1, 0, 7, 20, 0, 1, 0, 11, 20, 0, 1, 0, 8, 20, 0], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
expected = {
{ -- main
line = 1,
modifiers = { 'declaration', 'globalScope' },
start_col = 4,
end_col = 8,
type = 'function',
extmark_added = true,
},
{ -- __cplusplus
line = 3,
modifiers = { 'globalScope' },
start_col = 9,
end_col = 20,
type = 'macro',
extmark_added = true,
},
{ -- x
line = 4,
modifiers = { 'declaration', 'readonly', 'functionScope' },
start_col = 12,
end_col = 13,
type = 'variable',
extmark_added = true,
},
{ -- std
line = 5,
modifiers = { 'defaultLibrary', 'globalScope' },
start_col = 2,
end_col = 5,
type = 'namespace',
extmark_added = true,
},
{ -- cout
line = 5,
modifiers = { 'defaultLibrary', 'globalScope' },
start_col = 7,
end_col = 11,
type = 'variable',
extmark_added = true,
},
{ -- x
line = 5,
modifiers = { 'readonly', 'functionScope' },
start_col = 15,
end_col = 16,
type = 'variable',
extmark_added = true,
},
{ -- std
line = 5,
modifiers = { 'defaultLibrary', 'globalScope' },
start_col = 20,
end_col = 23,
type = 'namespace',
extmark_added = true,
},
{ -- endl
line = 5,
modifiers = { 'defaultLibrary', 'globalScope' },
start_col = 25,
end_col = 29,
type = 'function',
extmark_added = true,
},
{ -- #else comment #endif
line = 6,
modifiers = {},
start_col = 0,
end_col = 7,
type = 'comment',
extmark_added = true,
},
{
line = 7,
modifiers = {},
start_col = 0,
end_col = 11,
type = 'comment',
extmark_added = true,
},
{
line = 8,
modifiers = {},
start_col = 0,
end_col = 8,
type = 'comment',
extmark_added = true,
},
},
},
{
it = 'sumneko_lua',
text = [[-- comment
local a = 1
b = "as"]],
response = [[{"data": [0, 0, 10, 17, 0, 1, 6, 1, 8, 1, 1, 0, 1, 8, 8]}]],
legend = [[{
"tokenTypes": [
"namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "variable", "property", "enumMember", "event", "function", "method", "macro", "keyword", "modifier", "comment", "string", "number", "regexp", "operator"
],
"tokenModifiers": [
"declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary"
]
}]],
expected = {
{
line = 0,
modifiers = {},
start_col = 0,
end_col = 10,
type = 'comment', -- comment
extmark_added = true,
},
{
line = 1,
modifiers = { 'declaration' }, -- a
start_col = 6,
end_col = 7,
type = 'variable',
extmark_added = true,
},
{
line = 2,
modifiers = { 'static' }, -- b (global)
start_col = 0,
end_col = 1,
type = 'variable',
extmark_added = true,
},
},
},
{
it = 'rust-analyzer',
text = [[pub fn main() {
break rust;
/// what?
}
]] ,
response = [[{"data": [0, 0, 3, 1, 0, 0, 4, 2, 1, 0, 0, 3, 4, 14, 524290, 0, 4, 1, 45, 0, 0, 1, 1, 45, 0, 0, 2, 1, 26, 0, 1, 4, 5, 1, 8192, 0, 6, 4, 52, 0, 0, 4, 1, 48, 0, 1, 4, 9, 0, 1, 1, 0, 1, 26, 0], "resultId": "1"}]],
legend = [[{
"tokenTypes": [
"comment", "keyword", "string", "number", "regexp", "operator", "namespace", "type", "struct", "class", "interface", "enum", "enumMember", "typeParameter", "function", "method", "property", "macro", "variable",
"parameter", "angle", "arithmetic", "attribute", "attributeBracket", "bitwise", "boolean", "brace", "bracket", "builtinAttribute", "builtinType", "character", "colon", "comma", "comparison", "constParameter", "derive",
"dot", "escapeSequence", "formatSpecifier", "generic", "label", "lifetime", "logical", "macroBang", "operator", "parenthesis", "punctuation", "selfKeyword", "semicolon", "typeAlias", "toolModule", "union", "unresolvedReference"
],
"tokenModifiers": [
"documentation", "declaration", "definition", "static", "abstract", "deprecated", "readonly", "defaultLibrary", "async", "attribute", "callable", "constant", "consuming", "controlFlow", "crateRoot", "injected", "intraDocLink",
"library", "mutable", "public", "reference", "trait", "unsafe"
]
}]],
expected = {
{
line = 0,
modifiers = {},
start_col = 0,
end_col = 3, -- pub
type = 'keyword',
extmark_added = true,
},
{
line = 0,
modifiers = {},
start_col = 4,
end_col = 6, -- fn
type = 'keyword',
extmark_added = true,
},
{
line = 0,
modifiers = { 'declaration', 'public' },
start_col = 7,
end_col = 11, -- main
type = 'function',
extmark_added = true,
},
{
line = 0,
modifiers = {},
start_col = 11,
end_col = 12,
type = 'parenthesis',
extmark_added = true,
},
{
line = 0,
modifiers = {},
start_col = 12,
end_col = 13,
type = 'parenthesis',
extmark_added = true,
},
{
line = 0,
modifiers = {},
start_col = 14,
end_col = 15,
type = 'brace',
extmark_added = true,
},
{
line = 1,
modifiers = { 'controlFlow' },
start_col = 4,
end_col = 9, -- break
type = 'keyword',
extmark_added = true,
},
{
line = 1,
modifiers = {},
start_col = 10,
end_col = 13, -- rust
type = 'unresolvedReference',
extmark_added = true,
},
{
line = 1,
modifiers = {},
start_col = 13,
end_col = 13,
type = 'semicolon',
extmark_added = true,
},
{
line = 2,
modifiers = { 'documentation' },
start_col = 4,
end_col = 11,
type = 'comment', -- /// what?
extmark_added = true,
},
{
line = 3,
modifiers = {},
start_col = 0,
end_col = 1,
type = 'brace',
extmark_added = true,
},
},
},
}) do
it(test.it, function()
exec_lua(create_server_definition)
exec_lua([[
local legend, resp = ...
server = _create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function()
return vim.fn.json_decode(resp)
end,
}
})
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
]], test.legend, test.response)
insert(test.text)
local highlights = exec_lua([[
local semantic_tokens = vim.lsp.semantic_tokens
return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
]])
eq(test.expected, highlights)
end)
end
end)
describe('token decoding with deltas', function()
for _, test in ipairs({
{
it = 'semantic_tokens_delta: clangd-15 on C',
name = 'semantic_tokens_delta',
legend = [[{
"tokenTypes": [
"variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
],
"tokenModifiers": [
"declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
]
}]],
text = [[char* foo = "\n";]],
edit = [[ggO<Esc>]],
response1 = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]],
response2 = [[{"edits": [{ "start": 0, "deleteCount": 1, "data": [1] }], "resultId": "2"}]],
expected1 = {
{
line = 0,
modifiers = {
'declaration',
'globalScope',
},
start_col = 6,
end_col = 9,
type = 'variable',
extmark_added = true,
}
},
expected2 = {
{
line = 1,
modifiers = {
'declaration',
'globalScope',
},
start_col = 6,
end_col = 9,
type = 'variable',
extmark_added = true,
}
},
}
}) do
it(test.it, function()
exec_lua(create_server_definition)
exec_lua([[
local legend, resp1, resp2 = ...
server = _create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = true },
legend = vim.fn.json_decode(legend),
},
},
handlers = {
['textDocument/semanticTokens/full'] = function()
return vim.fn.json_decode(resp1)
end,
['textDocument/semanticTokens/full/delta'] = function()
return vim.fn.json_decode(resp2)
end,
}
})
bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
semantic_tokens = vim.lsp.semantic_tokens
]], test.legend, test.response1, test.response2)
insert(test.text)
local highlights = exec_lua([[
return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
]])
eq(test.expected1, highlights)
feed(test.edit)
highlights = exec_lua([[
return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
]])
eq(test.expected2, highlights)
end)
end
end)
end)