mirrors/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2026-03-30 20:32:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(lsp): support range + full semantic token requests #37611

Browse Source 
From the LSP Spec:
> There are two uses cases where it can be beneficial to only compute
> semantic tokens for a visible range:
>
> - for faster rendering of the tokens in the user interface when a user
>   opens a file. In this use case, servers should also implement the
>   textDocument/semanticTokens/full request as well to allow for flicker
>   free scrolling and semantic coloring of a minimap.
> - if computing semantic tokens for a full document is too expensive,
>   servers can only provide a range call. In this case, the client might
>   not render a minimap correctly or might even decide to not show any
>   semantic tokens at all.

This commit unifies the usage of range and full/delta requests as
recommended by the LSP spec and aligns neovim with the way other LSP
clients use these request types for semantic tokens.

When a server supports range requests, neovim will simultaneously send a
range request and a full/delta request when first opening a file, and
will continue to issue range requests until a full response is
processed. At that point, range requests cease and full (or delta)
requests are used going forward. The range request should allow servers
to return a result faster for quicker highlighting of the file while it
works on the potentially more expensive full result. If a server decides
the full result is too expensive, it can just error out that request,
and neovim will continue to use range requests.

This commit also fixes and cleans up some other things:

- gen_lsp: registrationMethod or registrationOptions imply dynamic
  registration support
- move autocmd creation/deletion to on_attach/on_detach
- debounce requests due to server refresh notifications
- fix off by one issue in tokens_to_ranges() iteration
This commit is contained in:
jdrouhard
2026-02-03 12:16:12 -06:00
committed by GitHub
parent fede568692
commit 9278f792c3
7 changed files with 259 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
runtime/doc/news.txt
View File

@@ -299,6 +299,7 @@ LSP
• Support for dynamic registration for `textDocument/diagnostic`
• |vim.lsp.buf.rename()| now highlights the symbol being renamed using the
|hl-LspReferenceTarget| highlight group.
• Support for `textDocument/semanticTokens/range`.
• Support for `textDocument/codeLens` |lsp-codelens| has been reimplemented:
https://microsoft.github.io/language-server-protocol/specifications/lsp/3.18/specification/#textDocument_codeLens
• Support for `workspace/codeLens/refresh`:
@@ -355,8 +356,6 @@ PERFORMANCE
additional constraints for improved correctness and resistance to
backtracking edge cases.
• |i_CTRL-R| inserts named/clipboard registers literally, 10x speedup.
• LSP `textDocument/semanticTokens/range` is supported which requests tokens
for the viewport (visible screen) only.
PLUGINS

14
runtime/lua/vim/lsp/_meta/protocol.lua
View File

@@ -1605,13 +1605,7 @@ error('Cannot require a meta file')
---@class lsp.DocumentOnTypeFormattingRegistrationOptions: lsp.TextDocumentRegistrationOptions, lsp.DocumentOnTypeFormattingOptions
---The parameters of a {@link RenameRequest}.
---@class lsp.RenameParams: lsp.WorkDoneProgressParams
---
---The document to rename.
---@field textDocument lsp.TextDocumentIdentifier
---
---The position at which this request was sent.
---@field position lsp.Position
---@class lsp.RenameParams: lsp.TextDocumentPositionParams, lsp.WorkDoneProgressParams
---
---The new name of the symbol. If the given name is not valid the
---request must return a {@link ResponseError} with an
@@ -3297,7 +3291,7 @@ error('Cannot require a meta file')
---@class lsp.FileOperationPattern
---
---The glob pattern to match. Glob patterns can have the following syntax:
---- `*` to match zero or more characters in a path segment
---- `*` to match one or more characters in a path segment
---- `?` to match on one character in a path segment
---- `**` to match any number of path segments, including none
---- `{}` to group sub patterns into an OR expression. (e.g. `**/*.{ts,js}` matches all TypeScript and JavaScript files)
@@ -5699,7 +5693,7 @@ error('Cannot require a meta file')
---its resource, or a glob-pattern that is applied to the {@link TextDocument.fileName path}.
---
---Glob patterns can have the following syntax:
---- `*` to match zero or more characters in a path segment
---- `*` to match one or more characters in a path segment
---- `?` to match on one character in a path segment
---- `**` to match any number of path segments, including none
---- `{}` to group sub patterns into an OR expression. (e.g. `**/*.{ts,js}` matches all TypeScript and JavaScript files)
@@ -5720,7 +5714,7 @@ error('Cannot require a meta file')
---@alias lsp.NotebookDocumentFilter lsp.NotebookDocumentFilterNotebookType|lsp.NotebookDocumentFilterScheme|lsp.NotebookDocumentFilterPattern
---The glob pattern to watch relative to the base path. Glob patterns can have the following syntax:
---- `*` to match zero or more characters in a path segment
---- `*` to match one or more characters in a path segment
---- `?` to match on one character in a path segment
---- `**` to match any number of path segments, including none
---- `{}` to group conditions (e.g. `**/*.{ts,js}` matches all TypeScript and JavaScript files)

2
runtime/lua/vim/lsp/client.lua
View File

@@ -1153,7 +1153,7 @@ function Client:on_attach(bufnr)
self:_run_callbacks(self._on_attach_cbs, lsp.client_errors.ON_ATTACH_ERROR, self, bufnr)
-- schedule the initialization of capabilities to give the above
-- on_attach and LspAttach callbacks the ability to schedule wrap the
-- opt-out (deleting the semanticTokensProvider from capabilities)
-- opt-out (such as deleting the semanticTokensProvider from capabilities)
vim.schedule(function()
if not vim.api.nvim_buf_is_valid(bufnr) then
return

1
runtime/lua/vim/lsp/protocol.lua
View File

@@ -1344,6 +1344,7 @@ protocol._request_name_allows_registration = {
['textDocument/selectionRange'] = true,
['textDocument/semanticTokens/full'] = true,
['textDocument/semanticTokens/full/delta'] = true,
['textDocument/semanticTokens/range'] = true,
['textDocument/signatureHelp'] = true,
['textDocument/typeDefinition'] = true,
['textDocument/willSave'] = true,

305
runtime/lua/vim/lsp/semantic_tokens.lua
View File

@@ -33,7 +33,9 @@ local M = {}
--- @field supports_range boolean
--- @field supports_delta boolean
--- @field active_request STActiveRequest
--- @field active_range_request STActiveRequest
--- @field current_result STCurrentResult
--- @field has_full_result boolean
---@class (private) STHighlighter : vim.lsp.Capability
---@field active table<integer, STHighlighter>
@@ -159,7 +161,7 @@ local function tokens_to_ranges(data, bufnr, client, request, ranges)
marked = false,
}
if last_insert_idx <= #ranges then
if last_insert_idx < #ranges then
local needs_insert = true
local idx = vim.list.bisect(ranges, { line = range.line }, {
lo = last_insert_idx,
@@ -231,36 +233,9 @@ function STHighlighter:new(bufnr)
end,
})
api.nvim_create_autocmd({ 'BufWinEnter', 'InsertLeave' }, {
buffer = self.bufnr,
group = self.augroup,
callback = function()
self:send_request()
end,
})
api.nvim_create_autocmd('WinScrolled', {
buffer = self.bufnr,
group = self.augroup,
callback = function()
self:on_change()
end,
})
return self
end
---@private
function STHighlighter:cancel_active_request(client_id)
local state = self.client_state[client_id]
if state.active_request.request_id then
local client = assert(vim.lsp.get_client_by_id(client_id))
client:cancel_request(state.active_request.request_id)
state.active_request.request_id = nil
state.active_request.version = nil
end
end
---@package
function STHighlighter:on_attach(client_id)
local client = vim.lsp.get_client_by_id(client_id)
@@ -275,10 +250,31 @@ function STHighlighter:on_attach(client_id)
and client:supports_method('textDocument/semanticTokens/full/delta', self.bufnr)
or false,
active_request = {},
active_range_request = {},
current_result = {},
has_full_result = false,
}
self.client_state[client_id] = state
end
api.nvim_create_autocmd({ 'BufWinEnter', 'InsertLeave' }, {
buffer = self.bufnr,
group = self.augroup,
callback = function()
self:send_request()
end,
})
if state.supports_range then
api.nvim_create_autocmd('WinScrolled', {
buffer = self.bufnr,
group = self.augroup,
callback = function()
self:on_change()
end,
})
end
self:send_request()
end
@@ -288,22 +284,26 @@ function STHighlighter:on_detach(client_id)
if state then
--TODO: delete namespace if/when that becomes possible
api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1)
api.nvim_clear_autocmds({ group = self.augroup })
self.client_state[client_id] = nil
end
end
--- This is the entry point for getting all the tokens in a buffer.
---
--- For the given clients (or all attached, if not provided), this sends a request
--- to ask for semantic tokens. If the server supports delta requests, that will
--- be prioritized if we have a previous requestId and token array.
--- For the given clients (or all attached, if not provided), this sends semantic token requests to
--- ask for semantic tokens. If the server supports range requests and a full result has not been
--- processed yet, it will send a range request for the current visible range. Additionally, if a
--- result for the current document version hasn't been processed yet, it sends either a full or
--- delta request, depending on what the server supports and whether there's a current full result
--- for the previous document version.
---
--- This function will skip servers where there is an already an active request in
--- flight for the same version. If there is a stale request in flight, that is
--- cancelled prior to sending a new one.
--- This function will skip full/delta requests on servers where there is an already an active
--- full/delta request in flight for the same version. If there is a stale request in flight, that
--- is cancelled prior to sending a new one.
---
--- Finally, if the request was successful, the requestId and document version
--- are saved to facilitate document synchronization in the response.
--- Finally, for successful requests, the requestId (full/delta) and document version are saved to
--- facilitate document synchronization in the response.
---
---@package
function STHighlighter:send_request()
@@ -314,66 +314,154 @@ function STHighlighter:send_request()
for client_id, state in pairs(self.client_state) do
local client = vim.lsp.get_client_by_id(client_id)
if client then
local current_result = state.current_result
local active_request = state.active_request
-- If the server supports range and there's no full result yet, then start with a range
-- request
if state.supports_range and not state.has_full_result then
self:send_range_request(client, state, version)
end
if
state.supports_range
or (current_result.version ~= version and active_request.version ~= version)
(not state.has_full_result or state.current_result.version ~= version)
and state.active_request.version ~= version
then
-- cancel stale in-flight request
if active_request.request_id then
client:cancel_request(active_request.request_id)
active_request.request_id = nil
active_request.version = nil
end
---@type lsp.SemanticTokensParams|lsp.SemanticTokensRangeParams|lsp.SemanticTokensDeltaParams
local params = { textDocument = util.make_text_document_params(self.bufnr) }
---@type vim.lsp.protocol.Method.ClientToServer.Request
local method = 'textDocument/semanticTokens/full'
if state.supports_range then
method = 'textDocument/semanticTokens/range'
params.range = self:get_visible_range()
elseif state.supports_delta and current_result.result_id then
method = 'textDocument/semanticTokens/full/delta'
params.previousResultId = current_result.result_id
end
---@param response? lsp.SemanticTokens|lsp.SemanticTokensDelta
local success, request_id = client:request(method, params, function(err, response, ctx)
local bufnr = assert(ctx.bufnr)
local highlighter = STHighlighter.active[bufnr]
if not highlighter then
return
end
if err or not response then
active_request.request_id = nil
active_request.version = nil
return
end
coroutine.wrap(STHighlighter.process_response)(
highlighter,
response,
client,
ctx.request_id,
version
)
end, self.bufnr)
if success then
active_request.request_id = request_id
active_request.version = version
end
self:send_full_delta_request(client, state, version)
end
end
end
end
--- Send a range request for the visible area
---
---@private
---@param client vim.lsp.Client
---@param state STClientState
---@param version integer
function STHighlighter:send_range_request(client, state, version)
local active_request = state.active_range_request
-- cancel stale in-flight request
if active_request and active_request.request_id then
client:cancel_request(active_request.request_id)
active_request.request_id = nil
active_request.version = nil
end
---@type lsp.SemanticTokensRangeParams
local params = {
textDocument = util.make_text_document_params(self.bufnr),
range = self:get_visible_range(),
}
---@type vim.lsp.protocol.Method.ClientToServer.Request
local method = 'textDocument/semanticTokens/range'
---@param response? lsp.SemanticTokens
local success, request_id = client:request(method, params, function(err, response, ctx)
local bufnr = assert(ctx.bufnr)
local highlighter = STHighlighter.active[bufnr]
if not highlighter then
return
end
-- Only process range response if we got a valid response and don't have a full result yet
if err or not response or state.has_full_result then
active_request.request_id = nil
active_request.version = nil
return
end
coroutine.wrap(STHighlighter.process_response)(
highlighter,
response,
client,
ctx.request_id,
version,
true
)
end, self.bufnr)
if success then
active_request.request_id = request_id
active_request.version = version
end
end
--- Send a full or delta request
---
---@private
---@param client vim.lsp.Client
---@param state STClientState
---@param version integer
function STHighlighter:send_full_delta_request(client, state, version)
local current_result = state.current_result
local active_request = state.active_request
-- cancel stale in-flight request
if active_request.request_id then
client:cancel_request(active_request.request_id)
active_request.request_id = nil
active_request.version = nil
end
---@type lsp.SemanticTokensParams|lsp.SemanticTokensDeltaParams
local params = { textDocument = util.make_text_document_params(self.bufnr) }
---@type vim.lsp.protocol.Method.ClientToServer.Request
local method = 'textDocument/semanticTokens/full'
if state.supports_delta and current_result.result_id then
method = 'textDocument/semanticTokens/full/delta'
params.previousResultId = current_result.result_id
end
---@param response? lsp.SemanticTokens|lsp.SemanticTokensDelta
local success, request_id = client:request(method, params, function(err, response, ctx)
local bufnr = assert(ctx.bufnr)
local highlighter = STHighlighter.active[bufnr]
if not highlighter then
return
end
if err or not response then
active_request.request_id = nil
active_request.version = nil
return
end
coroutine.wrap(STHighlighter.process_response)(
highlighter,
response,
client,
ctx.request_id,
version,
false
)
end, self.bufnr)
if success then
active_request.request_id = request_id
active_request.version = version
end
end
---@private
function STHighlighter:cancel_active_request(client_id)
local state = self.client_state[client_id]
local client = vim.lsp.get_client_by_id(client_id)
---@param request STActiveRequest
local function clear(request)
if client and request.request_id then
client:cancel_request(request.request_id)
request.request_id = nil
request.version = nil
end
end
clear(state.active_range_request)
clear(state.active_request)
end
--- Gets a range that encompasses all visible lines across all windows
--- @private
--- @return lsp.Range
@@ -418,15 +506,24 @@ end
---@param client vim.lsp.Client
---@param request_id integer
---@param version integer
---@param is_range_request boolean
---@private
function STHighlighter:process_response(response, client, request_id, version)
function STHighlighter:process_response(response, client, request_id, version, is_range_request)
local state = self.client_state[client.id]
if not state then
return
end
---@type STActiveRequest
local active_request
if is_range_request then
active_request = state.active_range_request
else
active_request = state.active_request
end
-- ignore stale responses
if state.active_request.request_id and request_id ~= state.active_request.request_id then
if active_request.request_id and request_id ~= active_request.request_id then
return
end
@@ -467,15 +564,20 @@ function STHighlighter:process_response(response, client, request_id, version)
-- convert token list to highlight ranges
-- this could yield and run over multiple event loop iterations
highlights = tokens_to_ranges(tokens, self.bufnr, client, state.active_request, highlights)
highlights = tokens_to_ranges(tokens, self.bufnr, client, active_request, highlights)
-- if this was a full result, mark the state as having processed it
if not is_range_request then
state.has_full_result = true
end
-- reset active request
state.active_request.request_id = nil
state.active_request.version = nil
active_request.request_id = nil
active_request.version = nil
-- update the state with the new results
current_result.version = version
current_result.result_id = response.resultId
current_result.result_id = not is_range_request and response.resultId or nil
current_result.tokens = tokens
current_result.highlights = highlights
if version_changed then
@@ -633,6 +735,7 @@ function STHighlighter:reset()
for client_id, state in pairs(self.client_state) do
api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1)
state.current_result = {}
state.has_full_result = false
self:cancel_active_request(client_id)
end
end
@@ -647,14 +750,17 @@ end
function STHighlighter:mark_dirty(client_id)
local state = assert(self.client_state[client_id])
-- if we clear the version from current_result, it'll cause the
-- next request to be sent and will also pause new highlights
-- from being added in on_win until a new result comes from
-- the server
-- if we clear the version from current_result, it'll cause the next
-- full/delta request to be sent and will also pause new highlights
-- from being added in on_win until a new result comes from the server
if state.current_result then
state.current_result.version = nil
end
-- clearing this flag will also allow range requests to fire to
-- potentially get a faster result
state.has_full_result = false
self:cancel_active_request(client_id)
end
@@ -944,7 +1050,8 @@ function M._refresh(err, _, ctx)
highlighter:mark_dirty(ctx.client_id)
if not vim.tbl_isempty(vim.fn.win_findbuf(bufnr)) then
highlighter:send_request()
-- some LSPs send rapid fire refresh notifications, so we'll debounce them with on_change()
highlighter:on_change()
end
end
end

2
src/gen/gen_lsp.lua
View File

@@ -292,7 +292,7 @@ local function write_to_vim_protocol(protocol)
})
for _, item in ipairs(all) do
if item.registrationOptions then
if item.registrationMethod or item.registrationOptions then
output[#output + 1] = (" ['%s'] = %s,"):format(item.method, true)
end
end

93
test/functional/plugin/lsp/semantic_tokens_spec.lua
View File

@@ -71,17 +71,17 @@ describe('semantic token highlighting', function()
local response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1024, 1, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ],
"resultId": 1
"resultId": "1"
}]]
local range_response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025 ],
"resultId": "1"
"resultId": "2"
}]]
local edit_response = [[{
"edits": [ {"data": [ 2, 8, 1, 3, 8193, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 3, 8192 ], "deleteCount": 25, "start": 5 } ],
"resultId":"2"
"resultId": "3"
}]]
before_each(function()
@@ -104,7 +104,7 @@ describe('semantic token highlighting', function()
end,
},
})
end, legend, response, edit_response)
end)
end)
it('buffer is highlighted when attached', function()
@@ -142,7 +142,7 @@ describe('semantic token highlighting', function()
_G.server2 = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = true },
full = { delta = false },
legend = vim.fn.json_decode(legend),
},
},
@@ -183,14 +183,14 @@ describe('semantic token highlighting', function()
}
end)
it('does not call full when only range is supported', function()
it('calls both range and full when range is supported', function()
insert(text)
exec_lua(function()
_G.server_range_only = _G._create_server({
_G.server_range = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = false },
range = true,
full = false,
legend = vim.fn.json_decode(legend),
},
},
@@ -203,11 +203,11 @@ describe('semantic token highlighting', function()
end,
},
})
end, legend, range_response)
end)
exec_lua(function()
local bufnr = vim.api.nvim_get_current_buf()
vim.api.nvim_win_set_buf(0, bufnr)
vim.lsp.start({ name = 'dummy', cmd = _G.server_range_only.cmd })
vim.lsp.start({ name = 'dummy', cmd = _G.server_range.cmd })
end)
screen:expect {
@@ -217,11 +217,11 @@ describe('semantic token highlighting', function()
int {8:main}() |
{ |
int {7:x}; |
#ifdef __cplusplus |
std::cout << x << "\n"; |
#else |
printf("%d\n", x); |
#endif |
#ifdef {5:__cplusplus} |
{4:std}::{2:cout} << {2:x} << "\n"; |
{6:#else} |
{6: printf("%d\n", x);} |
{6:#endif} |
} |
^} |
{1:~ }|*3
@@ -229,7 +229,7 @@ describe('semantic token highlighting', function()
]],
}
local messages = exec_lua('return server_range_only.messages')
local messages = exec_lua('return _G.server_range.messages')
local called_range = false
local called_full = false
for _, m in ipairs(messages) do
@@ -241,7 +241,7 @@ describe('semantic token highlighting', function()
end
end
eq(true, called_range)
eq(false, called_full)
eq(true, called_full)
end)
it('does not call range when only full is supported', function()
@@ -266,9 +266,9 @@ describe('semantic token highlighting', function()
},
})
return vim.lsp.start({ name = 'dummy', cmd = _G.server_full.cmd })
end, legend, response, range_response)
end)
local messages = exec_lua('return server_full.messages')
local messages = exec_lua('return _G.server_full.messages')
local called_full = false
local called_range = false
for _, m in ipairs(messages) do
@@ -283,14 +283,14 @@ describe('semantic token highlighting', function()
eq(false, called_range)
end)
it('prefers range when both are supported', function()
it('does not call range after full request received', function()
exec_lua(create_server_definition)
insert(text)
exec_lua(function()
_G.server_full = _G._create_server({
capabilities = {
semanticTokensProvider = {
full = { delta = true },
full = { delta = false },
range = true,
legend = vim.fn.json_decode(legend),
},
@@ -305,21 +305,24 @@ describe('semantic token highlighting', function()
},
})
return vim.lsp.start({ name = 'dummy', cmd = _G.server_full.cmd })
end, legend, response, range_response)
end)
local messages = exec_lua('return server_full.messages')
local called_full = false
local called_range = false
-- modify the buffer
feed('o<ESC>')
local messages = exec_lua('return _G.server_full.messages')
local called_full = 0
local called_range = 0
for _, m in ipairs(messages) do
if m.method == 'textDocument/semanticTokens/full' then
called_full = true
called_full = called_full + 1
end
if m.method == 'textDocument/semanticTokens/range' then
called_range = true
called_range = called_range + 1
end
end
eq(false, called_full)
eq(true, called_range)
eq(2, called_full)
eq(1, called_range)
end)
it('range requests preserve highlights outside updated range', function()
@@ -327,17 +330,15 @@ describe('semantic token highlighting', function()
insert(text)
feed('gg')
local small_range_response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025 ]
}]]
local client_id, bufnr = exec_lua(function(l, resp)
_G.response = resp
local client_id, bufnr = exec_lua(function()
_G.response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025 ]
}]]
_G.server2 = _G._create_server({
capabilities = {
semanticTokensProvider = {
range = true,
legend = vim.fn.json_decode(l),
legend = vim.fn.json_decode(legend),
},
},
handlers = {
@@ -349,10 +350,10 @@ describe('semantic token highlighting', function()
local bufnr = vim.api.nvim_get_current_buf()
local client_id = assert(vim.lsp.start({ name = 'dummy', cmd = _G.server2.cmd }))
vim.schedule(function()
vim.lsp.semantic_tokens._start(bufnr, client_id, 10)
vim.lsp.semantic_tokens._start(bufnr, client_id, 0)
end)
return client_id, bufnr
end, legend, small_range_response)
end)
screen:expect {
grid = [[
@@ -372,13 +373,11 @@ describe('semantic token highlighting', function()
end)
)
small_range_response = [[{
exec_lua(function()
_G.response = [[{
"data": [ 7, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ]
}]]
exec_lua(function(resp)
_G.response = resp
end, small_range_response)
end)
feed('G')
@@ -400,13 +399,11 @@ describe('semantic token highlighting', function()
end)
)
small_range_response = [[{
exec_lua(function()
_G.response = [[{
"data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192 ]
}]]
exec_lua(function(resp)
_G.response = resp
end, small_range_response)
end)
feed('ggLj0')
screen:expect {