Merge pull request #21100 from jdrouhard/lsp_semantic_tokens

LSP: semantic tokens support

runtime/doc/lsp.txt

@@ -1319,6 +1319,55 @@ save({lenses}, {bufnr}, {client_id})                 *vim.lsp.codelens.save()*
       • {client_id}  (number)
 
 
+==============================================================================
+Lua module: vim.lsp.semantic_tokens                      *lsp-semantic_tokens*
+
+force_refresh({bufnr})               *vim.lsp.semantic_tokens.force_refresh()*
+    Force a refresh of all semantic tokens
+
+    Only has an effect if the buffer is currently active for semantic token
+    highlighting (|vim.lsp.semantic_tokens.start()| has been called for it)
+
+    Parameters: ~
+      • {bufnr}  (nil|number) default: current buffer
+
+start({bufnr}, {client_id}, {opts})          *vim.lsp.semantic_tokens.start()*
+    Start the semantic token highlighting engine for the given buffer with the
+    given client. The client must already be attached to the buffer.
+
+    NOTE: This is currently called automatically by
+    |vim.lsp.buf_attach_client()|. To opt-out of semantic highlighting with a
+    server that supports it, you can delete the semanticTokensProvider table
+    from the {server_capabilities} of your client in your |LspAttach| callback
+    or your configuration's `on_attach` callback.
+
+    >lua
+
+       client.server_capabilities.semanticTokensProvider = nil
+<
+
+    Parameters: ~
+      • {bufnr}      (number)
+      • {client_id}  (number)
+      • {opts}       (nil|table) Optional keyword arguments
+                     • debounce (number, default: 200): Debounce token
+                       requests to the server by the given number in
+                       milliseconds
+
+stop({bufnr}, {client_id})                    *vim.lsp.semantic_tokens.stop()*
+    Stop the semantic token highlighting engine for the given buffer with the
+    given client.
+
+    NOTE: This is automatically called by a |LspDetach| autocmd that is set up
+    as part of `start()`, so you should only need this function to manually
+    disengage the semantic token engine without fully detaching the LSP client
+    from the buffer.
+
+    Parameters: ~
+      • {bufnr}      (number)
+      • {client_id}  (number)
+
+
 ==============================================================================
 Lua module: vim.lsp.handlers                                    *lsp-handlers*
 

runtime/doc/lua.txt

@@ -604,6 +604,7 @@ vim.highlight.priorities                            *vim.highlight.priorities*
     Table with default priorities used for highlighting:
         • `syntax`: `50`, used for standard syntax highlighting
         • `treesitter`: `100`, used for tree-sitter-based highlighting
+        • `semantic_tokens`: `125`, used for LSP semantic token highlighting
         • `diagnostics`: `150`, used for code analysis such as diagnostics
         • `user`: `200`, used for user-triggered highlights such as LSP document
           symbols or `on_yank` autocommands

runtime/doc/news.txt

@@ -39,6 +39,14 @@ NEW FEATURES                                                    *news-features*
 
 The following new APIs or features were added.
 
+• Added support for semantic token highlighting to the LSP client. This
+  functionality is enabled by default when a client that supports this feature
+  is attached to a buffer. Opt-out can be performed by deleting the
+  `semanticTokensProvider` from the LSP client's {server_capabilities} in the
+  `LspAttach` callback.
+
+  See |lsp-semantic_tokens| for more information.
+
 • |vim.treesitter.show_tree()| opens a split window showing a text
   representation of the nodes in a language tree for the current buffer.
 

runtime/lua/vim/highlight.lua

@@ -5,6 +5,7 @@ local M = {}
 M.priorities = {
   syntax = 50,
   treesitter = 100,
+  semantic_tokens = 125,
   diagnostics = 150,
   user = 200,
 }

runtime/lua/vim/lsp.lua

@@ -4,6 +4,7 @@ local lsp_rpc = require('vim.lsp.rpc')
 local protocol = require('vim.lsp.protocol')
 local util = require('vim.lsp.util')
 local sync = require('vim.lsp.sync')
+local semantic_tokens = require('vim.lsp.semantic_tokens')
 
 local api = vim.api
 local nvim_err_writeln, nvim_buf_get_lines, nvim_command, nvim_buf_get_option, nvim_exec_autocmds =
@@ -25,6 +26,7 @@ local lsp = {
   buf = require('vim.lsp.buf'),
   diagnostic = require('vim.lsp.diagnostic'),
   codelens = require('vim.lsp.codelens'),
+  semantic_tokens = semantic_tokens,
   util = util,
 
   -- Allow raw RPC access.
@@ -56,6 +58,8 @@ lsp._request_name_to_capability = {
   ['textDocument/formatting'] = { 'documentFormattingProvider' },
   ['textDocument/completion'] = { 'completionProvider' },
   ['textDocument/documentHighlight'] = { 'documentHighlightProvider' },
+  ['textDocument/semanticTokens/full'] = { 'semanticTokensProvider' },
+  ['textDocument/semanticTokens/full/delta'] = { 'semanticTokensProvider' },
 }
 
 -- TODO improve handling of scratch buffers with LSP attached.
@@ -1526,6 +1530,11 @@ function lsp.start_client(config)
       -- TODO(ashkan) handle errors.
       pcall(config.on_attach, client, bufnr)
     end
+
+    if vim.tbl_get(client.server_capabilities, 'semanticTokensProvider', 'full') then
+      semantic_tokens.start(bufnr, client.id)
+    end
+
     client.attached_buffers[bufnr] = true
   end
 

runtime/lua/vim/lsp/protocol.lua

@@ -629,6 +629,58 @@ export interface WorkspaceClientCapabilities {
 function protocol.make_client_capabilities()
   return {
     textDocument = {
+      semanticTokens = {
+        dynamicRegistration = false,
+        tokenTypes = {
+          'namespace',
+          'type',
+          'class',
+          'enum',
+          'interface',
+          'struct',
+          'typeParameter',
+          'parameter',
+          'variable',
+          'property',
+          'enumMember',
+          'event',
+          'function',
+          'method',
+          'macro',
+          'keyword',
+          'modifier',
+          'comment',
+          'string',
+          'number',
+          'regexp',
+          'operator',
+          'decorator',
+        },
+        tokenModifiers = {
+          'declaration',
+          'definition',
+          'readonly',
+          'static',
+          'deprecated',
+          'abstract',
+          'async',
+          'modification',
+          'documentation',
+          'defaultLibrary',
+        },
+        formats = { 'relative' },
+        requests = {
+          -- TODO(jdrouhard): Add support for this
+          range = false,
+          full = { delta = true },
+        },
+
+        overlappingTokenSupport = true,
+        -- TODO(jdrouhard): Add support for this
+        multilineTokenSupport = false,
+        serverCancelSupport = false,
+        augmentsSyntaxTokens = true,
+      },
       synchronization = {
         dynamicRegistration = false,
 
@@ -772,6 +824,9 @@ function protocol.make_client_capabilities()
       workspaceEdit = {
         resourceOperations = { 'rename', 'create', 'delete' },
       },
+      semanticTokens = {
+        refreshSupport = true,
+      },
     },
     callHierarchy = {
       dynamicRegistration = false,

runtime/lua/vim/lsp/semantic_tokens.lua

@@ -0,0 +1,644 @@
+local api = vim.api
+local handlers = require('vim.lsp.handlers')
+local util = require('vim.lsp.util')
+
+--- @class STTokenRange
+--- @field line number line number 0-based
+--- @field start_col number start column 0-based
+--- @field end_col number end column 0-based
+--- @field type string token type as string
+--- @field modifiers string[] token modifiers as strings
+--- @field extmark_added boolean whether this extmark has been added to the buffer yet
+---
+--- @class STCurrentResult
+--- @field version number document version associated with this result
+--- @field result_id string resultId from the server; used with delta requests
+--- @field highlights STTokenRange[] cache of highlight ranges for this document version
+--- @field tokens number[] raw token array as received by the server. used for calculating delta responses
+--- @field namespace_cleared boolean whether the namespace was cleared for this result yet
+---
+--- @class STActiveRequest
+--- @field request_id number the LSP request ID of the most recent request sent to the server
+--- @field version number the document version associated with the most recent request
+---
+--- @class STClientState
+--- @field namespace number
+--- @field active_request STActiveRequest
+--- @field current_result STCurrentResult
+
+---@class STHighlighter
+---@field active table<number, STHighlighter>
+---@field bufnr number
+---@field augroup number augroup for buffer events
+---@field debounce number milliseconds to debounce requests for new tokens
+---@field timer table uv_timer for debouncing requests for new tokens
+---@field client_state table<number, STClientState>
+local STHighlighter = { active = {} }
+
+---@private
+local function binary_search(tokens, line)
+  local lo = 1
+  local hi = #tokens
+  while lo < hi do
+    local mid = math.floor((lo + hi) / 2)
+    if tokens[mid].line < line then
+      lo = mid + 1
+    else
+      hi = mid
+    end
+  end
+  return lo
+end
+
+--- Extracts modifier strings from the encoded number in the token array
+---
+---@private
+---@return string[]
+local function modifiers_from_number(x, modifiers_table)
+  ---@private
+  local function _get_bit(n, k)
+    --TODO(jdrouhard): remove once `bit` module is available for non-LuaJIT
+    if _G.bit then
+      return _G.bit.band(_G.bit.rshift(n, k), 1)
+    else
+      return math.floor((n / math.pow(2, k)) % 2)
+    end
+  end
+
+  local modifiers = {}
+  for i = 0, #modifiers_table - 1 do
+    local b = _get_bit(x, i)
+    if b == 1 then
+      modifiers[#modifiers + 1] = modifiers_table[i + 1]
+    end
+  end
+
+  return modifiers
+end
+
+--- Converts a raw token list to a list of highlight ranges used by the on_win callback
+---
+---@private
+---@return STTokenRange[]
+local function tokens_to_ranges(data, bufnr, client)
+  local legend = client.server_capabilities.semanticTokensProvider.legend
+  local token_types = legend.tokenTypes
+  local token_modifiers = legend.tokenModifiers
+  local ranges = {}
+
+  local line
+  local start_char = 0
+  for i = 1, #data, 5 do
+    local delta_line = data[i]
+    line = line and line + delta_line or delta_line
+    local delta_start = data[i + 1]
+    start_char = delta_line == 0 and start_char + delta_start or delta_start
+
+    -- data[i+3] +1 because Lua tables are 1-indexed
+    local token_type = token_types[data[i + 3] + 1]
+    local modifiers = modifiers_from_number(data[i + 4], token_modifiers)
+
+    ---@private
+    local function _get_byte_pos(char_pos)
+      return util._get_line_byte_from_position(bufnr, {
+        line = line,
+        character = char_pos,
+      }, client.offset_encoding)
+    end
+
+    local start_col = _get_byte_pos(start_char)
+    local end_col = _get_byte_pos(start_char + data[i + 2])
+
+    if token_type then
+      ranges[#ranges + 1] = {
+        line = line,
+        start_col = start_col,
+        end_col = end_col,
+        type = token_type,
+        modifiers = modifiers,
+        extmark_added = false,
+      }
+    end
+  end
+
+  return ranges
+end
+
+--- Construct a new STHighlighter for the buffer
+---
+---@private
+---@param bufnr number
+function STHighlighter.new(bufnr)
+  local self = setmetatable({}, { __index = STHighlighter })
+
+  self.bufnr = bufnr
+  self.augroup = api.nvim_create_augroup('vim_lsp_semantic_tokens:' .. bufnr, { clear = true })
+  self.client_state = {}
+
+  STHighlighter.active[bufnr] = self
+
+  api.nvim_buf_attach(bufnr, false, {
+    on_lines = function(_, buf)
+      local highlighter = STHighlighter.active[buf]
+      if not highlighter then
+        return true
+      end
+      highlighter:on_change()
+    end,
+    on_reload = function(_, buf)
+      local highlighter = STHighlighter.active[buf]
+      if highlighter then
+        highlighter:reset()
+        highlighter:send_request()
+      end
+    end,
+    on_detach = function(_, buf)
+      local highlighter = STHighlighter.active[buf]
+      if highlighter then
+        highlighter:destroy()
+      end
+    end,
+  })
+
+  api.nvim_create_autocmd({ 'BufWinEnter', 'InsertLeave' }, {
+    buffer = self.bufnr,
+    group = self.augroup,
+    callback = function()
+      self:send_request()
+    end,
+  })
+
+  api.nvim_create_autocmd('LspDetach', {
+    buffer = self.bufnr,
+    group = self.augroup,
+    callback = function(args)
+      self:detach(args.data.client_id)
+      if vim.tbl_isempty(self.client_state) then
+        self:destroy()
+      end
+    end,
+  })
+
+  return self
+end
+
+---@private
+function STHighlighter:destroy()
+  for client_id, _ in pairs(self.client_state) do
+    self:detach(client_id)
+  end
+
+  api.nvim_del_augroup_by_id(self.augroup)
+  STHighlighter.active[self.bufnr] = nil
+end
+
+---@private
+function STHighlighter:attach(client_id)
+  local state = self.client_state[client_id]
+  if not state then
+    state = {
+      namespace = api.nvim_create_namespace('vim_lsp_semantic_tokens:' .. client_id),
+      active_request = {},
+      current_result = {},
+    }
+    self.client_state[client_id] = state
+  end
+end
+
+---@private
+function STHighlighter:detach(client_id)
+  local state = self.client_state[client_id]
+  if state then
+    --TODO: delete namespace if/when that becomes possible
+    api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1)
+    self.client_state[client_id] = nil
+  end
+end
+
+--- This is the entry point for getting all the tokens in a buffer.
+---
+--- For the given clients (or all attached, if not provided), this sends a request
+--- to ask for semantic tokens. If the server supports delta requests, that will
+--- be prioritized if we have a previous requestId and token array.
+---
+--- This function will skip servers where there is an already an active request in
+--- flight for the same version. If there is a stale request in flight, that is
+--- cancelled prior to sending a new one.
+---
+--- Finally, if the request was successful, the requestId and document version
+--- are saved to facilitate document synchronization in the response.
+---
+---@private
+function STHighlighter:send_request()
+  local version = util.buf_versions[self.bufnr]
+
+  self:reset_timer()
+
+  for client_id, state in pairs(self.client_state) do
+    local client = vim.lsp.get_client_by_id(client_id)
+
+    local current_result = state.current_result
+    local active_request = state.active_request
+
+    -- Only send a request for this client if the current result is out of date and
+    -- there isn't a current a request in flight for this version
+    if client and current_result.version ~= version and active_request.version ~= version then
+      -- cancel stale in-flight request
+      if active_request.request_id then
+        client.cancel_request(active_request.request_id)
+        active_request = {}
+        state.active_request = active_request
+      end
+
+      local spec = client.server_capabilities.semanticTokensProvider.full
+      local hasEditProvider = type(spec) == 'table' and spec.delta
+
+      local params = { textDocument = util.make_text_document_params(self.bufnr) }
+      local method = 'textDocument/semanticTokens/full'
+
+      if hasEditProvider and current_result.result_id then
+        method = method .. '/delta'
+        params.previousResultId = current_result.result_id
+      end
+      local success, request_id = client.request(method, params, function(err, response, ctx)
+        -- look client up again using ctx.client_id instead of using a captured
+        -- client object
+        local c = vim.lsp.get_client_by_id(ctx.client_id)
+        local highlighter = STHighlighter.active[ctx.bufnr]
+        if not err and c and highlighter then
+          highlighter:process_response(response, c, version)
+        end
+      end, self.bufnr)
+
+      if success then
+        active_request.request_id = request_id
+        active_request.version = version
+      end
+    end
+  end
+end
+
+--- This function will parse the semantic token responses and set up the cache
+--- (current_result). It also performs document synchronization by checking the
+--- version of the document associated with the resulting request_id and only
+--- performing work if the response is not out-of-date.
+---
+--- Delta edits are applied if necessary, and new highlight ranges are calculated
+--- and stored in the buffer state.
+---
+--- Finally, a redraw command is issued to force nvim to redraw the screen to
+--- pick up changed highlight tokens.
+---
+---@private
+function STHighlighter:process_response(response, client, version)
+  local state = self.client_state[client.id]
+  if not state then
+    return
+  end
+
+  -- ignore stale responses
+  if state.active_request.version and version ~= state.active_request.version then
+    return
+  end
+
+  -- reset active request
+  state.active_request = {}
+
+  -- if we have a response to a delta request, update the state of our tokens
+  -- appropriately. if it's a full response, just use that
+  local tokens
+  local token_edits = response.edits
+  if token_edits then
+    table.sort(token_edits, function(a, b)
+      return a.start < b.start
+    end)
+
+    ---@private
+    local function _splice(list, start, remove_count, data)
+      local ret = vim.list_slice(list, 1, start)
+      vim.list_extend(ret, data)
+      vim.list_extend(ret, list, start + remove_count + 1)
+      return ret
+    end
+
+    tokens = state.current_result.tokens
+    for _, token_edit in ipairs(token_edits) do
+      tokens = _splice(tokens, token_edit.start, token_edit.deleteCount, token_edit.data)
+    end
+  else
+    tokens = response.data
+  end
+
+  -- Update the state with the new results
+  local current_result = state.current_result
+  current_result.version = version
+  current_result.result_id = response.resultId
+  current_result.tokens = tokens
+  current_result.highlights = tokens_to_ranges(tokens, self.bufnr, client)
+  current_result.namespace_cleared = false
+
+  api.nvim_command('redraw!')
+end
+
+--- on_win handler for the decoration provider (see |nvim_set_decoration_provider|)
+---
+--- If there is a current result for the buffer and the version matches the
+--- current document version, then the tokens are valid and can be applied. As
+--- the buffer is drawn, this function will add extmark highlights for every
+--- token in the range of visible lines. Once a highlight has been added, it
+--- sticks around until the document changes and there's a new set of matching
+--- highlight tokens available.
+---
+--- If this is the first time a buffer is being drawn with a new set of
+--- highlights for the current document version, the namespace is cleared to
+--- remove extmarks from the last version. It's done here instead of the response
+--- handler to avoid the "blink" that occurs due to the timing between the
+--- response handler and the actual redraw.
+---
+---@private
+function STHighlighter:on_win(topline, botline)
+  for _, state in pairs(self.client_state) do
+    local current_result = state.current_result
+    if current_result.version and current_result.version == util.buf_versions[self.bufnr] then
+      if not current_result.namespace_cleared then
+        api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1)
+        current_result.namespace_cleared = true
+      end
+
+      -- We can't use ephemeral extmarks because the buffer updates are not in
+      -- sync with the list of semantic tokens. There's a delay between the
+      -- buffer changing and when the LSP server can respond with updated
+      -- tokens, and we don't want to "blink" the token highlights while
+      -- updates are in flight, and we don't want to use stale tokens because
+      -- they likely won't line up right with the actual buffer.
+      --
+      -- Instead, we have to use normal extmarks that can attach to locations
+      -- in the buffer and are persisted between redraws.
+      local highlights = current_result.highlights
+      local idx = binary_search(highlights, topline)
+
+      for i = idx, #highlights do
+        local token = highlights[i]
+
+        if token.line > botline then
+          break
+        end
+
+        if not token.extmark_added then
+          -- `strict = false` is necessary here for the 1% of cases where the
+          -- current result doesn't actually match the buffer contents. Some
+          -- LSP servers can respond with stale tokens on requests if they are
+          -- still processing changes from a didChange notification.
+          --
+          -- LSP servers that do this _should_ follow up known stale responses
+          -- with a refresh notification once they've finished processing the
+          -- didChange notification, which would re-synchronize the tokens from
+          -- our end.
+          --
+          -- The server I know of that does this is clangd when the preamble of
+          -- a file changes and the token request is processed with a stale
+          -- preamble while the new one is still being built. Once the preamble
+          -- finishes, clangd sends a refresh request which lets the client
+          -- re-synchronize the tokens.
+          api.nvim_buf_set_extmark(self.bufnr, state.namespace, token.line, token.start_col, {
+            hl_group = '@' .. token.type,
+            end_col = token.end_col,
+            priority = vim.highlight.priorities.semantic_tokens,
+            strict = false,
+          })
+
+          --TODO(jdrouhard): do something with the modifiers
+
+          token.extmark_added = true
+        end
+      end
+    end
+  end
+end
+
+--- Reset the buffer's highlighting state and clears the extmark highlights.
+---
+---@private
+function STHighlighter:reset()
+  for client_id, state in pairs(self.client_state) do
+    api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1)
+    state.current_result = {}
+    if state.active_request.request_id then
+      local client = vim.lsp.get_client_by_id(client_id)
+      assert(client)
+      client.cancel_request(state.active_request.request_id)
+      state.active_request = {}
+    end
+  end
+end
+
+--- Mark a client's results as dirty. This method will cancel any active
+--- requests to the server and pause new highlights from being added
+--- in the on_win callback. The rest of the current results are saved
+--- in case the server supports delta requests.
+---
+---@private
+---@param client_id number
+function STHighlighter:mark_dirty(client_id)
+  local state = self.client_state[client_id]
+  assert(state)
+
+  -- if we clear the version from current_result, it'll cause the
+  -- next request to be sent and will also pause new highlights
+  -- from being added in on_win until a new result comes from
+  -- the server
+  if state.current_result then
+    state.current_result.version = nil
+  end
+
+  if state.active_request.request_id then
+    local client = vim.lsp.get_client_by_id(client_id)
+    assert(client)
+    client.cancel_request(state.active_request.request_id)
+    state.active_request = {}
+  end
+end
+
+---@private
+function STHighlighter:on_change()
+  self:reset_timer()
+  if self.debounce > 0 then
+    self.timer = vim.defer_fn(function()
+      self:send_request()
+    end, self.debounce)
+  else
+    self:send_request()
+  end
+end
+
+---@private
+function STHighlighter:reset_timer()
+  local timer = self.timer
+  if timer then
+    self.timer = nil
+    if not timer:is_closing() then
+      timer:stop()
+      timer:close()
+    end
+  end
+end
+
+local M = {}
+
+--- Start the semantic token highlighting engine for the given buffer with the
+--- given client. The client must already be attached to the buffer.
+---
+--- NOTE: This is currently called automatically by |vim.lsp.buf_attach_client()|. To
+--- opt-out of semantic highlighting with a server that supports it, you can
+--- delete the semanticTokensProvider table from the {server_capabilities} of
+--- your client in your |LspAttach| callback or your configuration's
+--- `on_attach` callback.
+---
+--- <pre>lua
+---   client.server_capabilities.semanticTokensProvider = nil
+--- </pre>
+---
+---@param bufnr number
+---@param client_id number
+---@param opts (nil|table) Optional keyword arguments
+---  - debounce (number, default: 200): Debounce token requests
+---        to the server by the given number in milliseconds
+function M.start(bufnr, client_id, opts)
+  vim.validate({
+    bufnr = { bufnr, 'n', false },
+    client_id = { client_id, 'n', false },
+  })
+
+  opts = opts or {}
+  assert(
+    (not opts.debounce or type(opts.debounce) == 'number'),
+    'opts.debounce must be a number with the debounce time in milliseconds'
+  )
+
+  local client = vim.lsp.get_client_by_id(client_id)
+  if not client then
+    vim.notify('[LSP] No client with id ' .. client_id, vim.log.levels.ERROR)
+    return
+  end
+
+  if not vim.lsp.buf_is_attached(bufnr, client_id) then
+    vim.notify(
+      '[LSP] Client with id ' .. client_id .. ' not attached to buffer ' .. bufnr,
+      vim.log.levels.WARN
+    )
+    return
+  end
+
+  if not vim.tbl_get(client.server_capabilities, 'semanticTokensProvider', 'full') then
+    vim.notify('[LSP] Server does not support semantic tokens', vim.log.levels.WARN)
+    return
+  end
+
+  local highlighter = STHighlighter.active[bufnr]
+
+  if not highlighter then
+    highlighter = STHighlighter.new(bufnr)
+    highlighter.debounce = opts.debounce or 200
+  else
+    highlighter.debounce = math.max(highlighter.debounce, opts.debounce or 200)
+  end
+
+  highlighter:attach(client_id)
+  highlighter:send_request()
+end
+
+--- Stop the semantic token highlighting engine for the given buffer with the
+--- given client.
+---
+--- NOTE: This is automatically called by a |LspDetach| autocmd that is set up as part
+--- of `start()`, so you should only need this function to manually disengage the semantic
+--- token engine without fully detaching the LSP client from the buffer.
+---
+---@param bufnr number
+---@param client_id number
+function M.stop(bufnr, client_id)
+  vim.validate({
+    bufnr = { bufnr, 'n', false },
+    client_id = { client_id, 'n', false },
+  })
+
+  local highlighter = STHighlighter.active[bufnr]
+  if not highlighter then
+    return
+  end
+
+  highlighter:detach(client_id)
+
+  if vim.tbl_isempty(highlighter.client_state) then
+    highlighter:destroy()
+  end
+end
+
+--- Force a refresh of all semantic tokens
+---
+--- Only has an effect if the buffer is currently active for semantic token
+--- highlighting (|vim.lsp.semantic_tokens.start()| has been called for it)
+---
+---@param bufnr (nil|number) default: current buffer
+function M.force_refresh(bufnr)
+  vim.validate({
+    bufnr = { bufnr, 'n', true },
+  })
+
+  if bufnr == nil or bufnr == 0 then
+    bufnr = api.nvim_get_current_buf()
+  end
+
+  local highlighter = STHighlighter.active[bufnr]
+  if not highlighter then
+    return
+  end
+
+  highlighter:reset()
+  highlighter:send_request()
+end
+
+--- |lsp-handler| for the method `workspace/semanticTokens/refresh`
+---
+--- Refresh requests are sent by the server to indicate a project-wide change
+--- that requires all tokens to be re-requested by the client. This handler will
+--- invalidate the current results of all buffers and automatically kick off a
+--- new request for buffers that are displayed in a window. For those that aren't, a
+--- the BufWinEnter event should take care of it next time it's displayed.
+---
+---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#semanticTokens_refreshRequest
+handlers['workspace/semanticTokens/refresh'] = function(err, _, ctx)
+  if err then
+    return vim.NIL
+  end
+
+  for _, bufnr in ipairs(vim.lsp.get_buffers_by_client_id(ctx.client_id)) do
+    local highlighter = STHighlighter.active[bufnr]
+    if highlighter and highlighter.client_state[ctx.client_id] then
+      highlighter:mark_dirty(ctx.client_id)
+
+      if not vim.tbl_isempty(vim.fn.win_findbuf(bufnr)) then
+        highlighter:send_request()
+      end
+    end
+  end
+
+  return vim.NIL
+end
+
+local namespace = api.nvim_create_namespace('vim_lsp_semantic_tokens')
+api.nvim_set_decoration_provider(namespace, {
+  on_win = function(_, _, bufnr, topline, botline)
+    local highlighter = STHighlighter.active[bufnr]
+    if highlighter then
+      highlighter:on_win(topline, botline)
+    end
+  end,
+})
+
+--- for testing only! there is no guarantee of API stability with this!
+---
+---@private
+M.__STHighlighter = STHighlighter
+
+return M

scripts/gen_vimdoc.py

@@ -183,6 +183,7 @@ CONFIG = {
             'diagnostic.lua',
             'codelens.lua',
             'tagfunc.lua',
+            'semantic_tokens.lua',
             'handlers.lua',
             'util.lua',
             'log.lua',

src/nvim/highlight_group.c

@@ -258,12 +258,23 @@ static const char *highlight_init_both[] = {
   "default link @type Type",
   "default link @type.definition Typedef",
   "default link @storageclass StorageClass",
-  "default link @structure Structure",
   "default link @namespace Identifier",
   "default link @include Include",
   "default link @preproc PreProc",
   "default link @debug Debug",
   "default link @tag Tag",
+
+  // LSP semantic tokens
+  "default link @class Structure",
+  "default link @struct Structure",
+  "default link @enum Type",
+  "default link @enumMember Constant",
+  "default link @event Identifier",
+  "default link @interface Identifier",
+  "default link @modifier Identifier",
+  "default link @regexp SpecialChar",
+  "default link @typeParameter Type",
+  "default link @decorator Identifier",
   NULL
 };
 

test/functional/plugin/lsp/helpers.lua

@@ -0,0 +1,178 @@
+local helpers = require('test.functional.helpers')(nil)
+
+local clear = helpers.clear
+local exec_lua = helpers.exec_lua
+local run = helpers.run
+local stop = helpers.stop
+local NIL = helpers.NIL
+
+local M = {}
+
+function M.clear_notrace()
+  -- problem: here be dragons
+  -- solution: don't look too closely for dragons
+  clear {env={
+    NVIM_LUA_NOTRACK="1";
+    VIMRUNTIME=os.getenv"VIMRUNTIME";
+  }}
+end
+
+M.create_server_definition = [[
+  function _create_server(opts)
+    opts = opts or {}
+    local server = {}
+    server.messages = {}
+
+    function server.cmd(dispatchers)
+      local closing = false
+      local handlers = opts.handlers or {}
+      local srv = {}
+
+      function srv.request(method, params, callback)
+        table.insert(server.messages, {
+          method = method,
+          params = params,
+        })
+        local handler = handlers[method]
+        if handler then
+          local response, err = handler(method, params)
+          if response then
+            callback(err, response)
+          end
+        elseif method == 'initialize' then
+          callback(nil, {
+            capabilities = opts.capabilities or {}
+          })
+        elseif method == 'shutdown' then
+          callback(nil, nil)
+        end
+        local request_id = #server.messages
+        return true, request_id
+      end
+
+      function srv.notify(method, params)
+        table.insert(server.messages, {
+          method = method,
+          params = params
+        })
+        if method == 'exit' then
+          dispatchers.on_exit(0, 15)
+        end
+      end
+
+      function srv.is_closing()
+        return closing
+      end
+
+      function srv.terminate()
+        closing = true
+      end
+
+      return srv
+    end
+
+    return server
+  end
+]]
+
+-- Fake LSP server.
+M.fake_lsp_code = 'test/functional/fixtures/fake-lsp-server.lua'
+M.fake_lsp_logfile = 'Xtest-fake-lsp.log'
+
+local function fake_lsp_server_setup(test_name, timeout_ms, options, settings)
+  exec_lua([=[
+    lsp = require('vim.lsp')
+    local test_name, fixture_filename, logfile, timeout, options, settings = ...
+    TEST_RPC_CLIENT_ID = lsp.start_client {
+      cmd_env = {
+        NVIM_LOG_FILE = logfile;
+        NVIM_LUA_NOTRACK = "1";
+      };
+      cmd = {
+        vim.v.progpath, '-Es', '-u', 'NONE', '--headless',
+        "-c", string.format("lua TEST_NAME = %q", test_name),
+        "-c", string.format("lua TIMEOUT = %d", timeout),
+        "-c", "luafile "..fixture_filename,
+      };
+      handlers = setmetatable({}, {
+        __index = function(t, method)
+          return function(...)
+            return vim.rpcrequest(1, 'handler', ...)
+          end
+        end;
+      });
+      workspace_folders = {{
+          uri = 'file://' .. vim.loop.cwd(),
+          name = 'test_folder',
+      }};
+      on_init = function(client, result)
+        TEST_RPC_CLIENT = client
+        vim.rpcrequest(1, "init", result)
+      end;
+      flags = {
+        allow_incremental_sync = options.allow_incremental_sync or false;
+        debounce_text_changes = options.debounce_text_changes or 0;
+      };
+      settings = settings;
+      on_exit = function(...)
+        vim.rpcnotify(1, "exit", ...)
+      end;
+    }
+  ]=], test_name, M.fake_lsp_code, M.fake_lsp_logfile, timeout_ms or 1e3, options or {}, settings or {})
+end
+
+function M.test_rpc_server(config)
+  if config.test_name then
+    M.clear_notrace()
+    fake_lsp_server_setup(config.test_name, config.timeout_ms or 1e3, config.options, config.settings)
+  end
+  local client = setmetatable({}, {
+    __index = function(_, name)
+      -- Workaround for not being able to yield() inside __index for Lua 5.1 :(
+      -- Otherwise I would just return the value here.
+      return function(...)
+        return exec_lua([=[
+        local name = ...
+        if type(TEST_RPC_CLIENT[name]) == 'function' then
+          return TEST_RPC_CLIENT[name](select(2, ...))
+        else
+          return TEST_RPC_CLIENT[name]
+        end
+        ]=], name, ...)
+      end
+    end;
+  })
+  local code, signal
+  local function on_request(method, args)
+    if method == "init" then
+      if config.on_init then
+        config.on_init(client, unpack(args))
+      end
+      return NIL
+    end
+    if method == 'handler' then
+      if config.on_handler then
+        config.on_handler(unpack(args))
+      end
+    end
+    return NIL
+  end
+  local function on_notify(method, args)
+    if method == 'exit' then
+      code, signal = unpack(args)
+      return stop()
+    end
+  end
+  --  TODO specify timeout?
+  --  run(on_request, on_notify, config.on_setup, 1000)
+  run(on_request, on_notify, config.on_setup)
+  if config.on_exit then
+    config.on_exit(code, signal)
+  end
+  stop()
+  if config.test_name then
+    exec_lua("vim.api.nvim_exec_autocmds('VimLeavePre', { modeline = false })")
+  end
+end
+
+return M

test/functional/plugin/lsp/semantic_tokens_spec.lua

@@ -0,0 +1,910 @@
+local helpers = require('test.functional.helpers')(after_each)
+local lsp_helpers = require('test.functional.plugin.lsp.helpers')
+local Screen = require('test.functional.ui.screen')
+
+local command = helpers.command
+local dedent = helpers.dedent
+local eq = helpers.eq
+local exec_lua = helpers.exec_lua
+local feed = helpers.feed
+local feed_command = helpers.feed_command
+local insert = helpers.insert
+local matches = helpers.matches
+
+local clear_notrace = lsp_helpers.clear_notrace
+local create_server_definition = lsp_helpers.create_server_definition
+
+before_each(function()
+  clear_notrace()
+end)
+
+after_each(function()
+  exec_lua("vim.api.nvim_exec_autocmds('VimLeavePre', { modeline = false })")
+end)
+
+describe('semantic token highlighting', function()
+
+  describe('general', function()
+    local text = dedent([[
+    #include <iostream>
+
+    int main()
+    {
+        int x;
+    #ifdef __cplusplus
+        std::cout << x << "\n";
+    #else
+        printf("%d\n", x);
+    #endif
+    }
+    }]])
+
+    local legend = [[{
+      "tokenTypes": [
+        "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
+      ],
+      "tokenModifiers": [
+        "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
+      ]
+    }]]
+
+    local response = [[{
+      "data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1024, 1, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ],
+      "resultId": 1
+    }]]
+
+    local edit_response = [[{
+      "edits": [ {"data": [ 2, 8, 1, 3, 8193, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 3, 8192 ], "deleteCount": 25, "start": 5 } ],
+      "resultId":"2"
+    }]]
+
+    local screen
+    before_each(function()
+      screen = Screen.new(40, 16)
+      screen:attach()
+      screen:set_default_attr_ids {
+        [1] = { bold = true, foreground = Screen.colors.Blue1 };
+        [2] = { foreground = Screen.colors.DarkCyan };
+        [3] = { foreground = Screen.colors.SlateBlue };
+        [4] = { bold = true, foreground = Screen.colors.SeaGreen };
+        [5] = { foreground = tonumber('0x6a0dad') };
+        [6] = { foreground = Screen.colors.Blue1 };
+      }
+      command([[ hi link @namespace Type ]])
+      command([[ hi link @function Special ]])
+
+      exec_lua(create_server_definition)
+      exec_lua([[
+        local legend, response, edit_response = ...
+        server = _create_server({
+          capabilities = {
+            semanticTokensProvider = {
+              full = { delta = true },
+              legend = vim.fn.json_decode(legend),
+            },
+          },
+          handlers = {
+            ['textDocument/semanticTokens/full'] = function()
+              return vim.fn.json_decode(response)
+            end,
+            ['textDocument/semanticTokens/full/delta'] = function()
+              return vim.fn.json_decode(edit_response)
+            end,
+          }
+        })
+      ]], legend, response, edit_response)
+    end)
+
+    it('buffer is highlighted when attached', function()
+      exec_lua([[
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+
+      insert(text)
+
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {3:main}()                              |
+        {                                       |
+            int {2:x};                              |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {2:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]] }
+    end)
+
+    it('buffer is unhighlighted when client is detached', function()
+      exec_lua([[
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+
+      insert(text)
+
+      exec_lua([[
+        vim.notify = function() end
+        vim.lsp.buf_detach_client(bufnr, client_id)
+      ]])
+
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int main()                              |
+        {                                       |
+            int x;                              |
+        #ifdef __cplusplus                      |
+            std::cout << x << "\n";             |
+        #else                                   |
+            printf("%d\n", x);                  |
+        #endif                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]] }
+    end)
+
+    it('buffer is highlighted and unhighlighted when semantic token highlighting is started and stopped'
+      , function()
+      exec_lua([[
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+
+      insert(text)
+
+      exec_lua([[
+        vim.notify = function() end
+        vim.lsp.semantic_tokens.stop(bufnr, client_id)
+      ]])
+
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int main()                              |
+        {                                       |
+            int x;                              |
+        #ifdef __cplusplus                      |
+            std::cout << x << "\n";             |
+        #else                                   |
+            printf("%d\n", x);                  |
+        #endif                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]] }
+
+      exec_lua([[
+        vim.lsp.semantic_tokens.start(bufnr, client_id)
+      ]])
+
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {3:main}()                              |
+        {                                       |
+            int {2:x};                              |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {2:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]] }
+    end)
+
+    it('buffer is re-highlighted when force refreshed', function()
+      exec_lua([[
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+
+      insert(text)
+
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {3:main}()                              |
+        {                                       |
+            int {2:x};                              |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {2:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]] }
+
+      exec_lua([[
+        vim.lsp.semantic_tokens.force_refresh(bufnr)
+      ]])
+
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {3:main}()                              |
+        {                                       |
+            int {2:x};                              |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {2:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]], unchanged = true }
+
+      local messages = exec_lua('return server.messages')
+      local token_request_count = 0
+      for _, message in ipairs(messages) do
+        assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received')
+        if message.method == 'textDocument/semanticTokens/full' then
+          token_request_count = token_request_count + 1
+        end
+      end
+      eq(2, token_request_count)
+    end)
+
+    it('destroys the highlighter if the buffer is deleted', function()
+      exec_lua([[
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+
+      insert(text)
+
+      local highlighters = exec_lua([[
+        vim.api.nvim_buf_delete(bufnr, { force = true })
+        local semantic_tokens = vim.lsp.semantic_tokens
+        return semantic_tokens.__STHighlighter.active
+      ]])
+
+      eq({}, highlighters)
+    end)
+
+    it('updates highlights with delta request on buffer change', function()
+      exec_lua([[
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+
+      insert(text)
+      feed_command('%s/int x/int x()/')
+      feed_command('noh')
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {3:main}()                              |
+        {                                       |
+            ^int {3:x}();                            |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {3:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        }                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+        :noh                                    |
+      ]] }
+    end)
+
+    it('prevents starting semantic token highlighting with invalid conditions', function()
+      exec_lua([[
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start_client({ name = 'dummy', cmd = server.cmd })
+        notifications = {}
+        vim.notify = function(...) table.insert(notifications, 1, {...}) end
+      ]])
+      eq(false, exec_lua("return vim.lsp.buf_is_attached(bufnr, client_id)"))
+
+      insert(text)
+
+      local notifications = exec_lua([[
+        vim.lsp.semantic_tokens.start(bufnr, client_id)
+        return notifications
+      ]])
+      matches('%[LSP%] Client with id %d not attached to buffer %d', notifications[1][1])
+
+      notifications = exec_lua([[
+        vim.lsp.semantic_tokens.start(bufnr, client_id + 1)
+        return notifications
+      ]])
+      matches('%[LSP%] No client with id %d', notifications[1][1])
+    end)
+
+    it('opt-out: does not activate semantic token highlighting if disabled in client attach',
+      function()
+        exec_lua([[
+          bufnr = vim.api.nvim_get_current_buf()
+          vim.api.nvim_win_set_buf(0, bufnr)
+          client_id = vim.lsp.start({
+            name = 'dummy',
+            cmd = server.cmd,
+            on_attach = function(client, bufnr)
+              client.server_capabilities.semanticTokensProvider = nil
+            end,
+          })
+        ]])
+        eq(true, exec_lua("return vim.lsp.buf_is_attached(bufnr, client_id)"))
+
+        insert(text)
+
+        screen:expect { grid = [[
+          #include <iostream>                     |
+                                                  |
+          int main()                              |
+          {                                       |
+              int x;                              |
+          #ifdef __cplusplus                      |
+              std::cout << x << "\n";             |
+          #else                                   |
+              printf("%d\n", x);                  |
+          #endif                                  |
+          }                                       |
+          ^}                                       |
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+                                                  |
+        ]] }
+
+        local notifications = exec_lua([[
+          local notifications = {}
+          vim.notify = function(...) table.insert(notifications, 1, {...}) end
+          vim.lsp.semantic_tokens.start(bufnr, client_id)
+          return notifications
+        ]])
+        eq('[LSP] Server does not support semantic tokens', notifications[1][1])
+
+        screen:expect { grid = [[
+          #include <iostream>                     |
+                                                  |
+          int main()                              |
+          {                                       |
+              int x;                              |
+          #ifdef __cplusplus                      |
+              std::cout << x << "\n";             |
+          #else                                   |
+              printf("%d\n", x);                  |
+          #endif                                  |
+          }                                       |
+          ^}                                       |
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+                                                  |
+          ]], unchanged = true }
+      end)
+
+    it('does not send delta requests if not supported by server', function()
+      exec_lua([[
+        local legend, response, edit_response = ...
+        server2 = _create_server({
+          capabilities = {
+            semanticTokensProvider = {
+              full = { delta = false },
+              legend = vim.fn.json_decode(legend),
+            },
+          },
+          handlers = {
+            ['textDocument/semanticTokens/full'] = function()
+              return vim.fn.json_decode(response)
+            end,
+            ['textDocument/semanticTokens/full/delta'] = function()
+              return vim.fn.json_decode(edit_response)
+            end,
+          }
+        })
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server2.cmd })
+      ]], legend, response, edit_response)
+
+      insert(text)
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {3:main}()                              |
+        {                                       |
+            int {2:x};                              |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {2:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]] }
+      feed_command('%s/int x/int x()/')
+      feed_command('noh')
+
+      -- the highlights don't change because our fake server sent the exact
+      -- same result for the same method (the full request). "x" would have
+      -- changed to highlight index 3 had we sent a delta request
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {3:main}()                              |
+        {                                       |
+            ^int {2:x}();                            |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {2:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        }                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+        :noh                                    |
+      ]] }
+      local messages = exec_lua('return server2.messages')
+      local token_request_count = 0
+      for _, message in ipairs(messages) do
+        assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received')
+        if message.method == 'textDocument/semanticTokens/full' then
+          token_request_count = token_request_count + 1
+        end
+      end
+      eq(2, token_request_count)
+    end)
+  end)
+
+  describe('token array decoding', function()
+    for _, test in ipairs({
+      {
+        it = 'clangd-15 on C',
+        text = [[char* foo = "\n";]],
+        response = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]],
+        legend = [[{
+          "tokenTypes": [
+            "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
+          ],
+          "tokenModifiers": [
+            "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
+          ]
+        }]],
+        expected = {
+          {
+            line = 0,
+            modifiers = {
+              'declaration',
+              'globalScope',
+            },
+            start_col = 6,
+            end_col = 9,
+            type = 'variable',
+            extmark_added = true,
+          },
+        },
+      },
+      {
+        it = 'clangd-15 on C++',
+        text = [[#include <iostream>
+int main()
+{
+  #ifdef __cplusplus
+  const int x = 1;
+  std::cout << x << std::endl;
+  #else
+    comment
+  #endif
+}]]      ,
+        response = [[{"data": [1, 4, 4, 3, 8193, 2, 9, 11, 19, 8192, 1, 12, 1, 1, 1033, 1, 2, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1032, 0, 5, 3, 15, 8448, 0, 5, 4, 3, 8448, 1, 0, 7, 20, 0, 1, 0, 11, 20, 0, 1, 0, 8, 20, 0], "resultId": "1"}]],
+        legend = [[{
+          "tokenTypes": [
+            "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
+          ],
+          "tokenModifiers": [
+            "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
+          ]
+        }]],
+        expected = {
+          { -- main
+            line = 1,
+            modifiers = { 'declaration', 'globalScope' },
+            start_col = 4,
+            end_col = 8,
+            type = 'function',
+            extmark_added = true,
+          },
+          { --  __cplusplus
+            line = 3,
+            modifiers = { 'globalScope' },
+            start_col = 9,
+            end_col = 20,
+            type = 'macro',
+            extmark_added = true,
+          },
+          { -- x
+            line = 4,
+            modifiers = { 'declaration', 'readonly', 'functionScope' },
+            start_col = 12,
+            end_col = 13,
+            type = 'variable',
+            extmark_added = true,
+          },
+          { -- std
+            line = 5,
+            modifiers = { 'defaultLibrary', 'globalScope' },
+            start_col = 2,
+            end_col = 5,
+            type = 'namespace',
+            extmark_added = true,
+          },
+          { -- cout
+            line = 5,
+            modifiers = { 'defaultLibrary', 'globalScope' },
+            start_col = 7,
+            end_col = 11,
+            type = 'variable',
+            extmark_added = true,
+          },
+          { -- x
+            line = 5,
+            modifiers = { 'readonly', 'functionScope' },
+            start_col = 15,
+            end_col = 16,
+            type = 'variable',
+            extmark_added = true,
+          },
+          { -- std
+            line = 5,
+            modifiers = { 'defaultLibrary', 'globalScope' },
+            start_col = 20,
+            end_col = 23,
+            type = 'namespace',
+            extmark_added = true,
+          },
+          { -- endl
+            line = 5,
+            modifiers = { 'defaultLibrary', 'globalScope' },
+            start_col = 25,
+            end_col = 29,
+            type = 'function',
+            extmark_added = true,
+          },
+          { -- #else comment #endif
+            line = 6,
+            modifiers = {},
+            start_col = 0,
+            end_col = 7,
+            type = 'comment',
+            extmark_added = true,
+          },
+          {
+            line = 7,
+            modifiers = {},
+            start_col = 0,
+            end_col = 11,
+            type = 'comment',
+            extmark_added = true,
+          },
+          {
+            line = 8,
+            modifiers = {},
+            start_col = 0,
+            end_col = 8,
+            type = 'comment',
+            extmark_added = true,
+          },
+        },
+      },
+      {
+        it = 'sumneko_lua',
+        text = [[-- comment
+local a = 1
+b = "as"]],
+        response = [[{"data": [0, 0, 10, 17, 0, 1, 6, 1, 8, 1, 1, 0, 1, 8, 8]}]],
+        legend = [[{
+          "tokenTypes": [
+            "namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "variable", "property", "enumMember", "event", "function", "method", "macro", "keyword", "modifier", "comment", "string", "number", "regexp", "operator"
+          ],
+          "tokenModifiers": [
+            "declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary"
+          ]
+        }]],
+        expected = {
+          {
+            line = 0,
+            modifiers = {},
+            start_col = 0,
+            end_col = 10,
+            type = 'comment', -- comment
+            extmark_added = true,
+          },
+          {
+            line = 1,
+            modifiers = { 'declaration' }, -- a
+            start_col = 6,
+            end_col = 7,
+            type = 'variable',
+            extmark_added = true,
+          },
+          {
+            line = 2,
+            modifiers = { 'static' }, -- b (global)
+            start_col = 0,
+            end_col = 1,
+            type = 'variable',
+            extmark_added = true,
+          },
+        },
+      },
+      {
+        it = 'rust-analyzer',
+        text = [[pub fn main() {
+  break rust;
+  /// what?
+}
+]]       ,
+        response = [[{"data": [0, 0, 3, 1, 0, 0, 4, 2, 1, 0, 0, 3, 4, 14, 524290, 0, 4, 1, 45, 0, 0, 1, 1, 45, 0, 0, 2, 1, 26, 0, 1, 4, 5, 1, 8192, 0, 6, 4, 52, 0, 0, 4, 1, 48, 0, 1, 4, 9, 0, 1, 1, 0, 1, 26, 0], "resultId": "1"}]],
+        legend = [[{
+        "tokenTypes": [
+          "comment", "keyword", "string", "number", "regexp", "operator", "namespace", "type", "struct", "class", "interface", "enum", "enumMember", "typeParameter", "function", "method", "property", "macro", "variable",
+          "parameter", "angle", "arithmetic", "attribute", "attributeBracket", "bitwise", "boolean", "brace", "bracket", "builtinAttribute", "builtinType", "character", "colon", "comma", "comparison", "constParameter", "derive",
+          "dot", "escapeSequence", "formatSpecifier", "generic", "label", "lifetime", "logical", "macroBang", "operator", "parenthesis", "punctuation", "selfKeyword", "semicolon", "typeAlias", "toolModule", "union", "unresolvedReference"
+        ],
+        "tokenModifiers": [
+          "documentation", "declaration", "definition", "static", "abstract", "deprecated", "readonly", "defaultLibrary", "async", "attribute", "callable", "constant", "consuming", "controlFlow", "crateRoot", "injected", "intraDocLink",
+          "library", "mutable", "public", "reference", "trait", "unsafe"
+        ]
+      }]],
+        expected = {
+          {
+            line = 0,
+            modifiers = {},
+            start_col = 0,
+            end_col = 3, -- pub
+            type = 'keyword',
+            extmark_added = true,
+          },
+          {
+            line = 0,
+            modifiers = {},
+            start_col = 4,
+            end_col = 6, -- fn
+            type = 'keyword',
+            extmark_added = true,
+          },
+          {
+            line = 0,
+            modifiers = { 'declaration', 'public' },
+            start_col = 7,
+            end_col = 11, -- main
+            type = 'function',
+            extmark_added = true,
+          },
+          {
+            line = 0,
+            modifiers = {},
+            start_col = 11,
+            end_col = 12,
+            type = 'parenthesis',
+            extmark_added = true,
+          },
+          {
+            line = 0,
+            modifiers = {},
+            start_col = 12,
+            end_col = 13,
+            type = 'parenthesis',
+            extmark_added = true,
+          },
+          {
+            line = 0,
+            modifiers = {},
+            start_col = 14,
+            end_col = 15,
+            type = 'brace',
+            extmark_added = true,
+          },
+          {
+            line = 1,
+            modifiers = { 'controlFlow' },
+            start_col = 4,
+            end_col = 9, -- break
+            type = 'keyword',
+            extmark_added = true,
+          },
+          {
+            line = 1,
+            modifiers = {},
+            start_col = 10,
+            end_col = 13, -- rust
+            type = 'unresolvedReference',
+            extmark_added = true,
+          },
+          {
+            line = 1,
+            modifiers = {},
+            start_col = 13,
+            end_col = 13,
+            type = 'semicolon',
+            extmark_added = true,
+          },
+          {
+            line = 2,
+            modifiers = { 'documentation' },
+            start_col = 4,
+            end_col = 11,
+            type = 'comment', -- /// what?
+            extmark_added = true,
+          },
+          {
+            line = 3,
+            modifiers = {},
+            start_col = 0,
+            end_col = 1,
+            type = 'brace',
+            extmark_added = true,
+          },
+        },
+      },
+    }) do
+      it(test.it, function()
+        exec_lua(create_server_definition)
+        exec_lua([[
+          local legend, resp = ...
+          server = _create_server({
+            capabilities = {
+              semanticTokensProvider = {
+                full = { delta = false },
+                legend = vim.fn.json_decode(legend),
+              },
+            },
+            handlers = {
+              ['textDocument/semanticTokens/full'] = function()
+                return vim.fn.json_decode(resp)
+              end,
+            }
+          })
+          bufnr = vim.api.nvim_get_current_buf()
+          vim.api.nvim_win_set_buf(0, bufnr)
+          client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+        ]], test.legend, test.response)
+
+        insert(test.text)
+
+        local highlights = exec_lua([[
+          local semantic_tokens = vim.lsp.semantic_tokens
+          return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
+        ]])
+        eq(test.expected, highlights)
+      end)
+    end
+  end)
+  describe('token decoding with deltas', function()
+    for _, test in ipairs({
+      {
+        it = 'semantic_tokens_delta: clangd-15 on C',
+        name = 'semantic_tokens_delta',
+        legend = [[{
+          "tokenTypes": [
+            "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment"
+          ],
+          "tokenModifiers": [
+            "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope"
+          ]
+        }]],
+        text = [[char* foo = "\n";]],
+        edit = [[ggO<Esc>]],
+        response1 = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]],
+        response2 = [[{"edits": [{ "start": 0, "deleteCount": 1, "data": [1] }], "resultId": "2"}]],
+        expected1 = {
+          {
+            line = 0,
+            modifiers = {
+              'declaration',
+              'globalScope',
+            },
+            start_col = 6,
+            end_col = 9,
+            type = 'variable',
+            extmark_added = true,
+          }
+        },
+        expected2 = {
+          {
+            line = 1,
+            modifiers = {
+              'declaration',
+              'globalScope',
+            },
+            start_col = 6,
+            end_col = 9,
+            type = 'variable',
+            extmark_added = true,
+          }
+        },
+      }
+    }) do
+      it(test.it, function()
+        exec_lua(create_server_definition)
+        exec_lua([[
+          local legend, resp1, resp2 = ...
+          server = _create_server({
+            capabilities = {
+              semanticTokensProvider = {
+                full = { delta = true },
+                legend = vim.fn.json_decode(legend),
+              },
+            },
+            handlers = {
+              ['textDocument/semanticTokens/full'] = function()
+                return vim.fn.json_decode(resp1)
+              end,
+              ['textDocument/semanticTokens/full/delta'] = function()
+                return vim.fn.json_decode(resp2)
+              end,
+            }
+          })
+          bufnr = vim.api.nvim_get_current_buf()
+          vim.api.nvim_win_set_buf(0, bufnr)
+          client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+          semantic_tokens = vim.lsp.semantic_tokens
+        ]], test.legend, test.response1, test.response2)
+
+        insert(test.text)
+
+        local highlights = exec_lua([[
+          return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
+        ]])
+
+        eq(test.expected1, highlights)
+
+        feed(test.edit)
+
+        highlights = exec_lua([[
+          return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
+        ]])
+
+        eq(test.expected2, highlights)
+      end)
+    end
+  end)
+end)

test/functional/plugin/lsp_spec.lua

@@ -1,8 +1,9 @@
 local helpers = require('test.functional.helpers')(after_each)
+local lsp_helpers = require('test.functional.plugin.lsp.helpers')
 
 local assert_log = helpers.assert_log
-local clear = helpers.clear
 local buf_lines = helpers.buf_lines
+local clear = helpers.clear
 local command = helpers.command
 local dedent = helpers.dedent
 local exec_lua = helpers.exec_lua
@@ -14,6 +15,7 @@ local pesc = helpers.pesc
 local insert = helpers.insert
 local funcs = helpers.funcs
 local retry = helpers.retry
+local stop = helpers.stop
 local NIL = helpers.NIL
 local read_file = require('test.helpers').read_file
 local write_file = require('test.helpers').write_file
@@ -22,186 +24,19 @@ local meths = helpers.meths
 local is_os = helpers.is_os
 local skip = helpers.skip
 
--- Use these to get access to a coroutine so that I can run async tests and use
+local clear_notrace = lsp_helpers.clear_notrace
--- yield.
+local create_server_definition = lsp_helpers.create_server_definition
-local run, stop = helpers.run, helpers.stop
+local fake_lsp_code = lsp_helpers.fake_lsp_code
+local fake_lsp_logfile = lsp_helpers.fake_lsp_logfile
+local test_rpc_server = lsp_helpers.test_rpc_server
 
 -- TODO(justinmk): hangs on Windows https://github.com/neovim/neovim/pull/11837
 if skip(is_os('win')) then return end
 
--- Fake LSP server.
-local fake_lsp_code = 'test/functional/fixtures/fake-lsp-server.lua'
-local fake_lsp_logfile = 'Xtest-fake-lsp.log'
-
 teardown(function()
   os.remove(fake_lsp_logfile)
 end)
 
-local function clear_notrace()
-  -- problem: here be dragons
-  -- solution: don't look for dragons to closely
-  clear {env={
-    NVIM_LUA_NOTRACK="1";
-    VIMRUNTIME=os.getenv"VIMRUNTIME";
-  }}
-end
-
-
-local create_server_definition = [[
-  function _create_server(opts)
-    opts = opts or {}
-    local server = {}
-    server.messages = {}
-
-    function server.cmd(dispatchers)
-      local closing = false
-      local handlers = opts.handlers or {}
-      local srv = {}
-
-      function srv.request(method, params, callback)
-        table.insert(server.messages, {
-          method = method,
-          params = params,
-        })
-        local handler = handlers[method]
-        if handler then
-          local response, err = handler(params)
-          if response then
-            callback(err, response)
-          end
-        elseif method == 'initialize' then
-          callback(nil, {
-            capabilities = opts.capabilities or {}
-          })
-        elseif method == 'shutdown' then
-          callback(nil, nil)
-        end
-        local request_id = #server.messages
-        return true, request_id
-      end
-
-      function srv.notify(method, params)
-        table.insert(server.messages, {
-          method = method,
-          params = params
-        })
-        if method == 'exit' then
-          dispatchers.on_exit(0, 15)
-        end
-      end
-
-      function srv.is_closing()
-        return closing
-      end
-
-      function srv.terminate()
-        closing = true
-      end
-
-      return srv
-    end
-
-    return server
-  end
-]]
-
-
-local function fake_lsp_server_setup(test_name, timeout_ms, options, settings)
-  exec_lua([=[
-    lsp = require('vim.lsp')
-    local test_name, fixture_filename, logfile, timeout, options, settings = ...
-    TEST_RPC_CLIENT_ID = lsp.start_client {
-      cmd_env = {
-        NVIM_LOG_FILE = logfile;
-        NVIM_LUA_NOTRACK = "1";
-      };
-      cmd = {
-        vim.v.progpath, '-Es', '-u', 'NONE', '--headless',
-        "-c", string.format("lua TEST_NAME = %q", test_name),
-        "-c", string.format("lua TIMEOUT = %d", timeout),
-        "-c", "luafile "..fixture_filename,
-      };
-      handlers = setmetatable({}, {
-        __index = function(t, method)
-          return function(...)
-            return vim.rpcrequest(1, 'handler', ...)
-          end
-        end;
-      });
-      workspace_folders = {{
-          uri = 'file://' .. vim.loop.cwd(),
-          name = 'test_folder',
-      }};
-      on_init = function(client, result)
-        TEST_RPC_CLIENT = client
-        vim.rpcrequest(1, "init", result)
-      end;
-      flags = {
-        allow_incremental_sync = options.allow_incremental_sync or false;
-        debounce_text_changes = options.debounce_text_changes or 0;
-      };
-      settings = settings;
-      on_exit = function(...)
-        vim.rpcnotify(1, "exit", ...)
-      end;
-    }
-  ]=], test_name, fake_lsp_code, fake_lsp_logfile, timeout_ms or 1e3, options or {}, settings or {})
-end
-
-local function test_rpc_server(config)
-  if config.test_name then
-    clear_notrace()
-    fake_lsp_server_setup(config.test_name, config.timeout_ms or 1e3, config.options, config.settings)
-  end
-  local client = setmetatable({}, {
-    __index = function(_, name)
-      -- Workaround for not being able to yield() inside __index for Lua 5.1 :(
-      -- Otherwise I would just return the value here.
-      return function(...)
-        return exec_lua([=[
-        local name = ...
-        if type(TEST_RPC_CLIENT[name]) == 'function' then
-          return TEST_RPC_CLIENT[name](select(2, ...))
-        else
-          return TEST_RPC_CLIENT[name]
-        end
-        ]=], name, ...)
-      end
-    end;
-  })
-  local code, signal
-  local function on_request(method, args)
-    if method == "init" then
-      if config.on_init then
-        config.on_init(client, unpack(args))
-      end
-      return NIL
-    end
-    if method == 'handler' then
-      if config.on_handler then
-        config.on_handler(unpack(args))
-      end
-    end
-    return NIL
-  end
-  local function on_notify(method, args)
-    if method == 'exit' then
-      code, signal = unpack(args)
-      return stop()
-    end
-  end
-  --  TODO specify timeout?
-  --  run(on_request, on_notify, config.on_setup, 1000)
-  run(on_request, on_notify, config.on_setup)
-  if config.on_exit then
-    config.on_exit(code, signal)
-  end
-  stop()
-  if config.test_name then
-    exec_lua("vim.api.nvim_exec_autocmds('VimLeavePre', { modeline = false })")
-  end
-end
-
 describe('LSP', function()
   before_each(function()
     clear_notrace()