feat(vimdoc): support Markdown code blocks (#25127)

Support Markdown code blocks in addition to <pre> blocks in Doxygen doc
comments.

Update doc comments in iter.lua as a test.

runtime/doc/develop.txt

@@ -192,7 +192,7 @@ Docstring format:
   `@note`, `@param`, `@returns`
 - Limited markdown is supported.
   - List-items start with `-` (useful to nest or "indent")
-- Use `<pre>` for code samples.
+- Use ``` for code samples.
   Code samples can be annotated as `vim` or `lua`
 
 Example: the help for |nvim_open_win()| is generated from a docstring defined
@@ -202,10 +202,16 @@ in src/nvim/api/win_config.c like this: >
     /// ...
     ///
     /// Example (Lua): window-relative float
-    /// <pre>lua
+    ///
-    ///     vim.api.nvim_open_win(0, false,
+    /// ```lua
-    ///       {relative='win', row=3, col=3, width=12, height=3})
+    /// vim.api.nvim_open_win(0, false, {
-    /// </pre>
+    ///   relative='win',
+    ///   row=3,
+    ///   col=3,
+    ///   width=12,
+    ///   height=3,
+    /// })
+    /// ```
     ///
     /// @param buffer Buffer to display
     /// @param enter  Enter the window

runtime/doc/lua.txt

@@ -3462,7 +3462,7 @@ Iter:next()                                                      *Iter:next()*
 
     Example: >lua
 
-     local it = vim.iter(string.gmatch('1 2 3', 'd+')):map(tonumber)
+        local it = vim.iter(string.gmatch('1 2 3', '%d+')):map(tonumber)
         it:next()
         -- 1
         it:next()
@@ -3670,7 +3670,7 @@ Iter:totable()                                                *Iter:totable()*
 
     Examples: >lua
 
-     vim.iter(string.gmatch('100 20 50', 'd+')):map(tonumber):totable()
+        vim.iter(string.gmatch('100 20 50', '%d+')):map(tonumber):totable()
         -- { 100, 20, 50 }
 
         vim.iter({ 1, 2, 3 }):map(function(v) return v, 2 * v end):totable()

runtime/lua/vim/iter.lua

@@ -15,7 +15,8 @@
 --- (for function iterators, this means that the first value returned by the function is nil).
 ---
 --- Examples:
---- <pre>lua
+---
+--- ```lua
 --- local it = vim.iter({ 1, 2, 3, 4, 5 })
 --- it:map(function(v)
 ---   return v * 3
@@ -52,7 +53,7 @@
 --- rb:push("b")
 --- vim.iter(rb):totable()
 --- -- { "a", "b" }
---- </pre>
+--- ```
 ---
 --- In addition to the |vim.iter()| function, the |vim.iter| module provides
 --- convenience functions like |vim.iter.filter()| and |vim.iter.totable()|.
@@ -140,9 +141,10 @@ end
 --- Add a filter step to the iterator pipeline.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 --- local bufs = vim.iter(vim.api.nvim_list_bufs()):filter(vim.api.nvim_buf_is_loaded)
---- </pre>
+--- ```
 ---
 ---@param f function(...):bool Takes all values returned from the previous stage
 ---                            in the pipeline and returns false or nil if the
@@ -176,7 +178,8 @@ end
 --- If the map function returns nil, the value is filtered from the iterator.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 --- local it = vim.iter({ 1, 2, 3, 4 }):map(function(v)
 ---   if v % 2 == 0 then
 ---     return v * 3
@@ -184,7 +187,7 @@ end
 --- end)
 --- it:totable()
 --- -- { 6, 12 }
---- </pre>
+--- ```
 ---
 ---@param f function(...):any Mapping function. Takes all values returned from
 ---                           the previous stage in the pipeline as arguments
@@ -288,7 +291,8 @@ end
 --- pipeline, each value will be included in a table.
 ---
 --- Examples:
---- <pre>lua
+---
+--- ```lua
 --- vim.iter(string.gmatch('100 20 50', '%d+')):map(tonumber):totable()
 --- -- { 100, 20, 50 }
 ---
@@ -297,7 +301,7 @@ end
 ---
 --- vim.iter({ a = 1, b = 2, c = 3 }):filter(function(k, v) return v % 2 ~= 0 end):totable()
 --- -- { { 'a', 1 }, { 'c', 3 } }
---- </pre>
+--- ```
 ---
 --- The generated table is a list-like table with consecutive, numeric indices.
 --- To create a map-like table with arbitrary keys, use |Iter:fold()|.
@@ -352,7 +356,8 @@ end
 --- Fold ("reduce") an iterator or table into a single value.
 ---
 --- Examples:
---- <pre>lua
+---
+--- ```lua
 --- -- Create a new table with only even values
 --- local t = { a = 1, b = 2, c = 3, d = 4 }
 --- local it = vim.iter(t)
@@ -362,7 +367,7 @@ end
 ---   return t
 --- end)
 --- -- { b = 2, d = 4 }
---- </pre>
+--- ```
 ---
 ---@generic A
 ---
@@ -398,7 +403,8 @@ end
 --- Return the next value from the iterator.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter(string.gmatch('1 2 3', '%d+')):map(tonumber)
 --- it:next()
@@ -408,7 +414,7 @@ end
 --- it:next()
 --- -- 3
 ---
---- </pre>
+--- ```
 ---
 ---@return any
 function Iter.next(self) -- luacheck: no unused args
@@ -431,13 +437,14 @@ end
 --- Only supported for iterators on list-like tables.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter({ 3, 6, 9, 12 }):rev()
 --- it:totable()
 --- -- { 12, 9, 6, 3 }
 ---
---- </pre>
+--- ```
 ---
 ---@return Iter
 function Iter.rev(self)
@@ -457,7 +464,8 @@ end
 --- Only supported for iterators on list-like tables.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter({ 3, 6, 9, 12 })
 --- it:peek()
@@ -467,7 +475,7 @@ end
 --- it:next()
 --- -- 3
 ---
---- </pre>
+--- ```
 ---
 ---@return any
 function Iter.peek(self) -- luacheck: no unused args
@@ -486,7 +494,8 @@ end
 --- Advances the iterator. Returns nil and drains the iterator if no value is found.
 ---
 --- Examples:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter({ 3, 6, 9, 12 })
 --- it:find(12)
@@ -500,7 +509,7 @@ end
 --- it:find(function(v) return v % 4 == 0 end)
 --- -- 12
 ---
---- </pre>
+--- ```
 ---
 ---@return any
 function Iter.find(self, f)
@@ -536,7 +545,8 @@ end
 --- Only supported for iterators on list-like tables.
 ---
 --- Examples:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter({ 1, 2, 3, 2, 1 }):enumerate()
 --- it:rfind(1)
@@ -544,7 +554,7 @@ end
 --- it:rfind(1)
 --- -- 1	1
 ---
---- </pre>
+--- ```
 ---
 ---@see Iter.find
 ---
@@ -578,13 +588,14 @@ end
 --- Only supported for iterators on list-like tables.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 --- local it = vim.iter({1, 2, 3, 4})
 --- it:nextback()
 --- -- 4
 --- it:nextback()
 --- -- 3
---- </pre>
+--- ```
 ---
 ---@return any
 function Iter.nextback(self) -- luacheck: no unused args
@@ -604,7 +615,8 @@ end
 --- Only supported for iterators on list-like tables.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 --- local it = vim.iter({1, 2, 3, 4})
 --- it:peekback()
 --- -- 4
@@ -612,7 +624,7 @@ end
 --- -- 4
 --- it:nextback()
 --- -- 4
---- </pre>
+--- ```
 ---
 ---@return any
 function Iter.peekback(self) -- luacheck: no unused args
@@ -629,13 +641,14 @@ end
 --- Skip values in the iterator.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter({ 3, 6, 9, 12 }):skip(2)
 --- it:next()
 --- -- 9
 ---
---- </pre>
+--- ```
 ---
 ---@param n number Number of values to skip.
 ---@return Iter
@@ -661,13 +674,14 @@ end
 --- Only supported for iterators on list-like tables.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 --- local it = vim.iter({ 1, 2, 3, 4, 5 }):skipback(2)
 --- it:next()
 --- -- 1
 --- it:nextback()
 --- -- 3
---- </pre>
+--- ```
 ---
 ---@param n number Number of values to skip.
 ---@return Iter
@@ -691,7 +705,8 @@ end
 --- This function advances the iterator.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter({ 3, 6, 9, 12 })
 --- it:nth(2)
@@ -699,7 +714,7 @@ end
 --- it:nth(2)
 --- -- 12
 ---
---- </pre>
+--- ```
 ---
 ---@param n number The index of the value to return.
 ---@return any
@@ -716,7 +731,8 @@ end
 --- Only supported for iterators on list-like tables.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter({ 3, 6, 9, 12 })
 --- it:nthback(2)
@@ -724,7 +740,7 @@ end
 --- it:nthback(2)
 --- -- 3
 ---
---- </pre>
+--- ```
 ---
 ---@param n number The index of the value to return.
 ---@return any
@@ -805,7 +821,8 @@ end
 --- Drains the iterator.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter(vim.gsplit('abcdefg', ''))
 --- it:last()
@@ -815,7 +832,7 @@ end
 --- it:last()
 --- -- 15
 ---
---- </pre>
+--- ```
 ---
 ---@return any
 function Iter.last(self)
@@ -839,19 +856,22 @@ end
 --- Add an iterator stage that returns the current iterator count as well as the iterator value.
 ---
 --- For list tables, prefer
---- <pre>lua
+---
+--- ```lua
 --- vim.iter(ipairs(t))
---- </pre>
+--- ```
 ---
 --- over
---- <pre>lua
+---
+--- ```lua
 --- vim.iter(t):enumerate()
---- </pre>
+--- ```
 ---
 --- as the former is faster.
 ---
 --- Example:
---- <pre>lua
+---
+--- ```lua
 ---
 --- local it = vim.iter(vim.gsplit('abc', '')):enumerate()
 --- it:next()
@@ -861,7 +881,7 @@ end
 --- it:next()
 --- -- 3	'c'
 ---
---- </pre>
+--- ```
 ---
 ---@return Iter
 function Iter.enumerate(self)
@@ -959,9 +979,10 @@ end
 --- Collect an iterator into a table.
 ---
 --- This is a convenience function that performs:
---- <pre>lua
+---
+--- ```lua
 --- vim.iter(f):totable()
---- </pre>
+--- ```
 ---
 ---@param f function Iterator function
 ---@return table
@@ -972,9 +993,10 @@ end
 --- Filter a table or iterator.
 ---
 --- This is a convenience function that performs:
---- <pre>lua
+---
+--- ```lua
 --- vim.iter(src):filter(f):totable()
---- </pre>
+--- ```
 ---
 ---@see |Iter:filter()|
 ---
@@ -990,9 +1012,10 @@ end
 --- Map and filter a table or iterator.
 ---
 --- This is a convenience function that performs:
---- <pre>lua
+---
+--- ```lua
 --- vim.iter(src):map(f):totable()
---- </pre>
+--- ```
 ---
 ---@see |Iter:map()|
 ---

scripts/gen_vimdoc.py

@@ -440,7 +440,7 @@ def is_blank(text):
     return '' == clean_lines(text)
 
 
-def get_text(n, preformatted=False):
+def get_text(n):
     """Recursively concatenates all text in a node tree."""
     text = ''
     if n.nodeType == n.TEXT_NODE:
@@ -449,11 +449,13 @@ def get_text(n, preformatted=False):
         for node in n.childNodes:
             text += get_text(node)
         return '`{}`'.format(text)
+    if n.nodeName == 'sp': # space, used in "programlisting" nodes
+        return ' '
     for node in n.childNodes:
         if node.nodeType == node.TEXT_NODE:
             text += node.data
         elif node.nodeType == node.ELEMENT_NODE:
-            text += get_text(node, preformatted)
+            text += get_text(node)
     return text
 
 
@@ -571,7 +573,7 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation,
     # text += (int(not space_preceding) * ' ')
 
     if n.nodeName == 'preformatted':
-        o = get_text(n, preformatted=True)
+        o = get_text(n)
         ensure_nl = '' if o[-1] == '\n' else '\n'
         if o[0:4] == 'lua\n':
             text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1])
@@ -581,7 +583,15 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation,
             text += o[4:-1]
         else:
             text += '>{}{}\n<'.format(ensure_nl, o)
+    elif n.nodeName == 'programlisting': # codeblock (```)
+        o = get_text(n)
+        filename = n.attributes['filename'].value
+        if filename:
+            text += '>{}'.format(filename.lstrip('.'))
+        else:
+            text += '>'
 
+        text += '\n\n{}\n<'.format(textwrap.indent(o, ' ' * 4))
     elif is_inline(n):
         text = doc_wrap(get_text(n), prefix=prefix, indent=indent, width=width)
     elif n.nodeName == 'verbatim':
@@ -786,6 +796,18 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=
     for child in parent.childNodes:
         para, _ = para_as_map(child, indent, width, fmt_vimhelp)
 
+        # 'programlisting' blocks are Markdown code blocks. Do not include
+        # these as a separate paragraph, but append to the last non-empty line
+        # in the text
+        if (
+            len(child.childNodes) == 1
+            and child.childNodes[0].nodeName == 'programlisting'
+        ):
+            while rendered_blocks and rendered_blocks[-1] == '':
+                rendered_blocks.pop()
+            rendered_blocks[-1] += ' ' + para['text']
+            continue
+
         # Generate text from the gathered items.
         chunks = [para['text']]
         if len(para['note']) > 0: