feat(lsp): use treesitter for stylize markdown

runtime/doc/lsp.txt

@@ -1678,20 +1678,24 @@ convert_input_to_markdown_lines({input}, {contents})
     window for `textDocument/hover`, for parsing the result of
     `textDocument/signatureHelp`, and potentially others.
 
+    Note that if the input is of type `MarkupContent` and its kind is
+    `plaintext`, then the corresponding value is returned without further
+    modifications.
+
     Parameters: ~
       • {input}     (`MarkedString` | `MarkedString[]` | `MarkupContent`)
       • {contents}  (table|nil) List of strings to extend with converted
                     lines. Defaults to {}.
 
     Return: ~
-        (table) {contents} extended with lines of converted markdown.
+        string[] extended with lines of converted markdown.
 
     See also: ~
       • https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_hover
 
                      *vim.lsp.util.convert_signature_help_to_markdown_lines()*
 convert_signature_help_to_markdown_lines({signature_help}, {ft}, {triggers})
-    Converts `textDocument/SignatureHelp` response to markdown lines.
+    Converts `textDocument/signatureHelp` response to markdown lines.
 
     Parameters: ~
       • {signature_help}  (table) Response of `textDocument/SignatureHelp`
@@ -1908,10 +1912,6 @@ open_floating_preview({contents}, {syntax}, {opts})
                       height when wrap is enabled
                     • max_width: (integer) maximal width of floating window
                     • max_height: (integer) maximal height of floating window
-                    • pad_top: (integer) number of lines to pad contents at
-                      top
-                    • pad_bottom: (integer) number of lines to pad contents at
-                      bottom
                     • focus_id: (string) if a popup with this id is opened,
                       then focus it
                     • close_events: (table) list of events that closes the
@@ -2005,8 +2005,6 @@ stylize_markdown({bufnr}, {contents}, {opts})
                     • wrap_at character to wrap at for computing height
                     • max_width maximal width of floating window
                     • max_height maximal height of floating window
-                    • pad_top number of lines to pad contents at top
-                    • pad_bottom number of lines to pad contents at bottom
                     • separator insert separator after code block
 
     Return: ~

runtime/doc/news.txt

@@ -232,6 +232,11 @@ The following changes to existing APIs or features add new behavior.
   In addition, |nvim_buf_get_extmarks()| has gained an "overlap" option to
   return such ranges even if they started before the specified position.
 
+• LSP hover and signature help now use Treesitter for highlighting of Markdown
+  content.
+  Note that syntax highlighting of code examples requires a matching parser
+  and may be affected by custom queries.
+
 ==============================================================================
 REMOVED FEATURES                                                 *news-removed*
 

runtime/lua/vim/lsp/handlers.lua

@@ -371,15 +371,22 @@ function M.hover(_, result, ctx, config)
     end
     return
   end
-  local markdown_lines = util.convert_input_to_markdown_lines(result.contents)
+  local format = 'markdown'
-  markdown_lines = util.trim_empty_lines(markdown_lines)
+  local contents ---@type string[]
-  if vim.tbl_isempty(markdown_lines) then
+  if type(result.contents) == 'table' and result.contents.kind == 'plaintext' then
+    format = 'plaintext'
+    contents = { result.contents.value or '' }
+  else
+    contents = util.convert_input_to_markdown_lines(result.contents)
+  end
+  contents = util.trim_empty_lines(contents)
+  if vim.tbl_isempty(contents) then
     if config.silent ~= true then
       vim.notify('No information available')
     end
     return
   end
-  return util.open_floating_preview(markdown_lines, 'markdown', config)
+  return util.open_floating_preview(contents, format, config)
 end
 
 --see: https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_hover

runtime/lua/vim/lsp/util.lua

@@ -877,9 +877,12 @@ end
 --- window for `textDocument/hover`, for parsing the result of
 --- `textDocument/signatureHelp`, and potentially others.
 ---
+--- Note that if the input is of type `MarkupContent` and its kind is `plaintext`,
+--- then the corresponding value is returned without further modifications.
+---
 ---@param input (`MarkedString` | `MarkedString[]` | `MarkupContent`)
 ---@param contents (table|nil) List of strings to extend with converted lines. Defaults to {}.
----@return table {contents} extended with lines of converted markdown.
+---@return string[] extended with lines of converted markdown.
 ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_hover
 function M.convert_input_to_markdown_lines(input, contents)
   contents = contents or {}
@@ -887,27 +890,13 @@ function M.convert_input_to_markdown_lines(input, contents)
   if type(input) == 'string' then
     list_extend(contents, split_lines(input))
   else
-    assert(type(input) == 'table', 'Expected a table for Hover.contents')
+    assert(type(input) == 'table', 'Expected a table for LSP input')
     -- MarkupContent
     if input.kind then
-      -- The kind can be either plaintext or markdown.
-      -- If it's plaintext, then wrap it in a <text></text> block
-
-      -- Some servers send input.value as empty, so let's ignore this :(
       local value = input.value or ''
-
-      if input.kind == 'plaintext' then
-        -- wrap this in a <text></text> block so that stylize_markdown
-        -- can properly process it as plaintext
-        value = string.format('<text>\n%s\n</text>', value)
-      end
-
-      -- assert(type(value) == 'string')
       list_extend(contents, split_lines(value))
       -- MarkupString variation 2
     elseif input.language then
-      -- Some servers send input.value as empty, so let's ignore this :(
-      -- assert(type(input.value) == 'string')
       table.insert(contents, '```' .. input.language)
       list_extend(contents, split_lines(input.value or ''))
       table.insert(contents, '```')
@@ -925,7 +914,7 @@ function M.convert_input_to_markdown_lines(input, contents)
   return contents
 end
 
---- Converts `textDocument/SignatureHelp` response to markdown lines.
+--- Converts `textDocument/signatureHelp` response to markdown lines.
 ---
 ---@param signature_help table Response of `textDocument/SignatureHelp`
 ---@param ft string|nil filetype that will be use as the `lang` for the label markdown code block
@@ -955,7 +944,7 @@ function M.convert_signature_help_to_markdown_lines(signature_help, ft, triggers
   end
   local label = signature.label
   if ft then
-    -- wrap inside a code block so stylize_markdown can render it properly
+    -- wrap inside a code block for proper rendering
     label = ('```%s\n%s\n```'):format(ft, label)
   end
   list_extend(contents, split(label, '\n', { plain = true }))
@@ -1223,7 +1212,7 @@ function M.preview_location(location, opts)
   local syntax = vim.bo[bufnr].syntax
   if syntax == '' then
     -- When no syntax is set, we use filetype as fallback. This might not result
-    -- in a valid syntax definition. See also ft detection in stylize_markdown.
+    -- in a valid syntax definition.
     -- An empty syntax is more common now with TreeSitter, since TS disables syntax.
     syntax = vim.bo[bufnr].filetype
   end
@@ -1240,36 +1229,65 @@ local function find_window_by_var(name, value)
   end
 end
 
---- Trims empty lines from input and pad top and bottom with empty lines
+---Returns true if the line is empty or only contains whitespace.
----
+---@param line string
----@param contents table of lines to trim and pad
+---@return boolean
----@param opts table with optional fields
+local function is_blank_line(line)
----             - pad_top    number of lines to pad contents at top (default 0)
+  return line and line:match('^%s*$')
----             - pad_bottom number of lines to pad contents at bottom (default 0)
+end
----@return table table of trimmed and padded lines
+
-function M._trim(contents, opts)
+---Returns true if the line corresponds to a Markdown thematic break.
-  validate({
+---@param line string
-    contents = { contents, 't' },
+---@return boolean
-    opts = { opts, 't', true },
+local function is_separator_line(line)
-  })
+  return line and line:match('^ ? ? ?%-%-%-+%s*$')
-  opts = opts or {}
+end
-  contents = M.trim_empty_lines(contents)
+
-  if opts.pad_top then
+---Replaces separator lines by the given divider and removing surrounding blank lines.
-    for _ = 1, opts.pad_top do
+---@param contents string[]
-      table.insert(contents, 1, '')
+---@param divider string
-    end
+---@return string[]
-  end
+local function replace_separators(contents, divider)
-  if opts.pad_bottom then
+  local trimmed = {}
-    for _ = 1, opts.pad_bottom do
+  local l = 1
-      table.insert(contents, '')
+  while l <= #contents do
-    end
+    local line = contents[l]
-  end
+    if is_separator_line(line) then
-  return contents
+      if l > 1 and is_blank_line(contents[l - 1]) then
+        table.remove(trimmed)
+      end
+      table.insert(trimmed, divider)
+      if is_blank_line(contents[l + 1]) then
+        l = l + 1
+      end
+    else
+      table.insert(trimmed, line)
+    end
+    l = l + 1
+  end
+
+  return trimmed
+end
+
+---Collapses successive blank lines in the input table into a single one.
+---@param contents string[]
+---@return string[]
+local function collapse_blank_lines(contents)
+  local collapsed = {}
+  local l = 1
+  while l <= #contents do
+    local line = contents[l]
+    if is_blank_line(line) then
+      while is_blank_line(contents[l + 1]) do
+        l = l + 1
+      end
+    end
+    table.insert(collapsed, line)
+    l = l + 1
+  end
+  return collapsed
 end
 
---- Generates a table mapping markdown code block lang to vim syntax,
---- based on g:markdown_fenced_languages
----@return table table of lang -> syntax mappings
 local function get_markdown_fences()
   local fences = {}
   for _, fence in pairs(vim.g.markdown_fenced_languages or {}) do
@@ -1297,8 +1315,6 @@ end
 ---  - wrap_at   character to wrap at for computing height
 ---  - max_width  maximal width of floating window
 ---  - max_height maximal height of floating window
----  - pad_top    number of lines to pad contents at top
----  - pad_bottom number of lines to pad contents at bottom
 ---  - separator insert separator after code block
 ---@return table stripped content
 function M.stylize_markdown(bufnr, contents, opts)
@@ -1335,7 +1351,7 @@ function M.stylize_markdown(bufnr, contents, opts)
   end
 
   -- Clean up
-  contents = M._trim(contents, opts)
+  contents = M.trim_empty_lines(contents)
 
   local stripped = {}
   local highlights = {}
@@ -1484,6 +1500,49 @@ function M.stylize_markdown(bufnr, contents, opts)
   return stripped
 end
 
+--- @class lsp.util.NormalizeMarkdownOptions
+--- @field width integer Thematic breaks are expanded to this size. Defaults to 80.
+
+--- Normalizes Markdown input to a canonical form.
+---
+--- The returned Markdown adheres to the GitHub Flavored Markdown (GFM)
+--- specification.
+---
+--- The following transformations are made:
+---
+---   1. Empty lines at the beginning or end of the content are removed
+---   2. Carriage returns ('\r') are removed
+---   3. Successive empty lines are collapsed into a single empty line
+---   4. Thematic breaks are expanded to the given width
+---
+---@private
+---@param contents string[]
+---@param opts? lsp.util.NormalizeMarkdownOptions
+---@return string[] table of lines containing normalized Markdown
+---@see https://github.github.com/gfm
+function M._normalize_markdown(contents, opts)
+  validate({
+    contents = { contents, 't' },
+    opts = { opts, 't', true },
+  })
+  opts = opts or {}
+
+  -- 1. Empty lines at the beginning or end of the content are removed
+  contents = M.trim_empty_lines(contents)
+
+  -- 2. Carriage returns are removed
+  contents = vim.split(table.concat(contents, '\n'):gsub('\r', ''), '\n')
+
+  -- 3. Successive empty lines are collapsed into a single empty line
+  contents = collapse_blank_lines(contents)
+
+  -- 4. Thematic breaks are expanded to the given width
+  local divider = string.rep('─', opts.width or 80)
+  contents = replace_separators(contents, divider)
+
+  return contents
+end
+
 --- Closes the preview window
 ---
 ---@param winnr integer window id of preview window
@@ -1620,8 +1679,6 @@ end
 ---             - wrap_at: (integer) character to wrap at for computing height when wrap is enabled
 ---             - max_width: (integer) maximal width of floating window
 ---             - max_height: (integer) maximal height of floating window
----             - pad_top: (integer) number of lines to pad contents at top
----             - pad_bottom: (integer) number of lines to pad contents at bottom
 ---             - focus_id: (string) if a popup with this id is opened, then focus it
 ---             - close_events: (table) list of events that closes the floating window
 ---             - focusable: (boolean, default true) Make float focusable
@@ -1629,8 +1686,7 @@ end
 ---                      is also `true`, focus an existing floating window with the same
 ---                      {focus_id}
 ---@return integer bufnr of newly created float window
----@return integer winid of newly created float window
+---@return integer winid of newly created float window preview window
----preview window
 function M.open_floating_preview(contents, syntax, opts)
   validate({
     contents = { contents, 't' },
@@ -1639,7 +1695,6 @@ function M.open_floating_preview(contents, syntax, opts)
   })
   opts = opts or {}
   opts.wrap = opts.wrap ~= false -- wrapping by default
-  opts.stylize_markdown = opts.stylize_markdown ~= false and vim.g.syntax_on ~= nil
   opts.focus = opts.focus ~= false
   opts.close_events = opts.close_events or { 'CursorMoved', 'CursorMovedI', 'InsertCharPre' }
 
@@ -1671,16 +1726,21 @@ function M.open_floating_preview(contents, syntax, opts)
     api.nvim_win_close(existing_float, true)
   end
 
+  -- Create the buffer
   local floating_bufnr = api.nvim_create_buf(false, true)
-  local do_stylize = syntax == 'markdown' and opts.stylize_markdown
-
-  -- Clean up input: trim empty lines from the end, pad
-  contents = M._trim(contents, opts)
 
+  -- Set up the contents, using treesitter for markdown
+  local do_stylize = syntax == 'markdown' and vim.g.syntax_on ~= nil
   if do_stylize then
-    -- applies the syntax and sets the lines to the buffer
+    local width = M._make_floating_popup_size(contents, opts)
-    contents = M.stylize_markdown(floating_bufnr, contents, opts)
+    contents = M._normalize_markdown(contents, { width = width })
+    vim.bo[floating_bufnr].filetype = 'markdown'
+    vim.treesitter.start(floating_bufnr)
+    api.nvim_buf_set_lines(floating_bufnr, 0, -1, false, contents)
   else
+    -- Clean up input: trim empty lines from the end, pad
+    contents = M.trim_empty_lines(contents)
+
     if syntax then
       vim.bo[floating_bufnr].syntax = syntax
     end
@@ -1697,9 +1757,9 @@ function M.open_floating_preview(contents, syntax, opts)
 
   local float_option = M.make_floating_popup_options(width, height, opts)
   local floating_winnr = api.nvim_open_win(floating_bufnr, false, float_option)
+
   if do_stylize then
     vim.wo[floating_winnr].conceallevel = 2
-    vim.wo[floating_winnr].concealcursor = 'n'
   end
   -- disable folding
   vim.wo[floating_winnr].foldenable = false
@@ -1708,6 +1768,7 @@ function M.open_floating_preview(contents, syntax, opts)
 
   vim.bo[floating_bufnr].modifiable = false
   vim.bo[floating_bufnr].bufhidden = 'wipe'
+
   api.nvim_buf_set_keymap(
     floating_bufnr,
     'n',

runtime/queries/markdown/highlights.scm

@@ -61,3 +61,11 @@
 ] @string.escape
 
 (inline) @spell
+
+;; Conceal backticks
+(fenced_code_block
+  (fenced_code_block_delimiter) @conceal
+  (#set! conceal ""))
+(fenced_code_block
+  (info_string (language) @conceal
+  (#set! conceal "")))

runtime/queries/markdown_inline/highlights.scm

@@ -92,3 +92,11 @@
     "]"
   ] @conceal
   (#set! conceal ""))
+
+;; Replace common HTML entities.
+((entity_reference) @conceal (#eq? @conceal " ") (#set! conceal ""))
+((entity_reference) @conceal (#eq? @conceal "&lt;") (#set! conceal "<"))
+((entity_reference) @conceal (#eq? @conceal "&gt;") (#set! conceal ">"))
+((entity_reference) @conceal (#eq? @conceal "&amp;") (#set! conceal "&"))
+((entity_reference) @conceal (#eq? @conceal "&quot;") (#set! conceal "\""))
+((entity_reference) @conceal (#any-of? @conceal "&ensp;" "&emsp;") (#set! conceal " "))

test/functional/plugin/lsp_spec.lua

@@ -3032,7 +3032,7 @@ describe('LSP', function()
         }
         return vim.lsp.util.convert_signature_help_to_markdown_lines(signature_help, 'cs', {','})
       ]]
-      local expected = {'```cs', 'TestEntity.TestEntity()', '```', '<text>', 'some doc', '</text>'}
+      local expected = {'```cs', 'TestEntity.TestEntity()', '```', 'some doc'}
       eq(expected, result)
     end)
   end)