feat(docs): replace lua2dox.lua

Problem:

The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.

Solution:

Re-implement the flow using Lua, LPEG and treesitter.

- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
  - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
  - LPEG for C parsing (see `scripts/cdoc_parser.lua`)
  - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
  - Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
   - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).

src/nvim/api/autocmd.c

@@ -75,7 +75,7 @@ static int64_t next_autocmd_id = 1;
 ///             - buffer: Buffer number or list of buffer numbers for buffer local autocommands
 ///             |autocmd-buflocal|. Cannot be used with {pattern}
 /// @return Array of autocommands matching the criteria, with each item
-///         containing the following fields:
+///             containing the following fields:
 ///             - id (number): the autocommand id (only when defined with the API).
 ///             - group (integer): the autocommand group id.
 ///             - group_name (string): the autocommand group name.
@@ -83,10 +83,10 @@ static int64_t next_autocmd_id = 1;
 ///             - event (string): the autocommand event.
 ///             - command (string): the autocommand command. Note: this will be empty if a callback is set.
 ///             - callback (function|string|nil): Lua function or name of a Vim script function
-///             which is executed when this autocommand is triggered.
+///               which is executed when this autocommand is triggered.
 ///             - once (boolean): whether the autocommand is only run once.
 ///             - pattern (string): the autocommand pattern.
-///             If the autocommand is buffer local |autocmd-buffer-local|:
+///               If the autocommand is buffer local |autocmd-buffer-local|:
 ///             - buflocal (boolean): true if the autocommand is buffer local.
 ///             - buffer (number): the buffer number.
 Array nvim_get_autocmds(Dict(get_autocmds) *opts, Arena *arena, Error *err)
@@ -536,9 +536,9 @@ void nvim_del_autocmd(Integer id, Error *err)
 /// @param opts Parameters
 ///         - event: (string|table)
 ///              Examples:
-///                 - event: "pat1"
-///                 - event: { "pat1" }
-///                 - event: { "pat1", "pat2", "pat3" }
+///              - event: "pat1"
+///              - event: { "pat1" }
+///              - event: { "pat1", "pat2", "pat3" }
 ///         - pattern: (string|table)
 ///             - pattern or patterns to match exactly.
 ///                 - For example, if you have `*.py` as that pattern for the autocmd,

src/nvim/api/buffer.c

@@ -50,9 +50,8 @@
 # include "api/buffer.c.generated.h"
 #endif
 
-/// \defgroup api-buffer
-///
-/// \brief For more information on buffers, see |buffers|
+/// @brief <pre>help
+/// For more information on buffers, see |buffers|.
 ///
 /// Unloaded Buffers: ~
 ///
@@ -64,6 +63,7 @@
 ///
 /// You can use |nvim_buf_is_loaded()| or |nvim_buf_line_count()| to check
 /// whether a buffer is loaded.
+/// </pre>
 
 /// Returns the number of lines in the given buffer.
 ///
@@ -229,6 +229,7 @@ Boolean nvim_buf_detach(uint64_t channel_id, Buffer buffer, Error *err)
   return true;
 }
 
+/// @nodoc
 void nvim__buf_redraw_range(Buffer buffer, Integer first, Integer last, Error *err)
 {
   buf_T *buf = find_buffer_by_handle(buffer, err);
@@ -875,8 +876,8 @@ Integer nvim_buf_get_changedtick(Buffer buffer, Error *err)
 
 /// Gets a list of buffer-local |mapping| definitions.
 ///
-/// @param  mode       Mode short-name ("n", "i", "v", ...)
 /// @param  buffer     Buffer handle, or 0 for current buffer
+/// @param  mode       Mode short-name ("n", "i", "v", ...)
 /// @param[out]  err   Error details, if any
 /// @returns Array of |maparg()|-like dictionaries describing mappings.
 ///          The "buffer" key holds the associated buffer handle.
@@ -1223,6 +1224,7 @@ Object nvim_buf_call(Buffer buffer, LuaRef fun, Error *err)
   return res;
 }
 
+/// @nodoc
 Dictionary nvim__buf_stats(Buffer buffer, Arena *arena, Error *err)
 {
   buf_T *buf = find_buffer_by_handle(buffer, err);

src/nvim/api/command.c

@@ -67,7 +67,7 @@
 ///             - file: (boolean) The command expands filenames. Which means characters such as "%",
 ///                               "#" and wildcards are expanded.
 ///             - bar: (boolean) The "|" character is treated as a command separator and the double
-///                              quote character (\") is treated as the start of a comment.
+///                              quote character (") is treated as the start of a comment.
 ///         - mods: (dictionary) |:command-modifiers|.
 ///             - filter: (dictionary) |:filter|.
 ///                 - pattern: (string) Filter pattern. Empty string if there is no filter.

src/nvim/api/deprecated.c

@@ -61,8 +61,7 @@ Object nvim_execute_lua(String code, Array args, Arena *arena, Error *err)
 
 /// Gets the buffer number
 ///
-/// @deprecated The buffer number now is equal to the object id,
-///             so there is no need to use this function.
+/// @deprecated The buffer number now is equal to the object id
 ///
 /// @param buffer     Buffer handle, or 0 for current buffer
 /// @param[out] err   Error details, if any
@@ -100,8 +99,7 @@ void nvim_buf_clear_highlight(Buffer buffer, Integer ns_id, Integer line_start,
 
 /// Set the virtual text (annotation) for a buffer line.
 ///
-/// @deprecated use nvim_buf_set_extmark to use full virtual text
-///             functionality.
+/// @deprecated use nvim_buf_set_extmark to use full virtual text functionality.
 ///
 /// The text will be placed after the buffer text. Virtual text will never
 /// cause reflow, rather virtual text will be truncated at the end of the screen
@@ -119,7 +117,7 @@ void nvim_buf_clear_highlight(Buffer buffer, Integer ns_id, Integer line_start,
 /// virtual text, the allocated id is then returned.
 ///
 /// @param buffer     Buffer handle, or 0 for current buffer
-/// @param ns_id      Namespace to use or 0 to create a namespace,
+/// @param src_id     Namespace to use or 0 to create a namespace,
 ///                   or -1 for a ungrouped annotation
 /// @param line       Line to annotate with virtual text (zero-indexed)
 /// @param chunks     A list of [text, hl_group] arrays, each representing a

src/nvim/api/extmark.c

@@ -1205,6 +1205,7 @@ free_exit:
   return virt_text;
 }
 
+/// @nodoc
 String nvim__buf_debug_extmarks(Buffer buffer, Boolean keys, Boolean dot, Error *err)
   FUNC_API_SINCE(7) FUNC_API_RET_ALLOC
 {

src/nvim/api/options.c

@@ -272,21 +272,21 @@ Dictionary nvim_get_all_options_info(Arena *arena, Error *err)
 /// Gets the option information for one option from arbitrary buffer or window
 ///
 /// Resulting dictionary has keys:
-///     - name: Name of the option (like 'filetype')
-///     - shortname: Shortened name of the option (like 'ft')
-///     - type: type of option ("string", "number" or "boolean")
-///     - default: The default value for the option
-///     - was_set: Whether the option was set.
+/// - name: Name of the option (like 'filetype')
+/// - shortname: Shortened name of the option (like 'ft')
+/// - type: type of option ("string", "number" or "boolean")
+/// - default: The default value for the option
+/// - was_set: Whether the option was set.
 ///
-///     - last_set_sid: Last set script id (if any)
-///     - last_set_linenr: line number where option was set
-///     - last_set_chan: Channel where option was set (0 for local)
+/// - last_set_sid: Last set script id (if any)
+/// - last_set_linenr: line number where option was set
+/// - last_set_chan: Channel where option was set (0 for local)
 ///
-///     - scope: one of "global", "win", or "buf"
-///     - global_local: whether win or buf option has a global value
+/// - scope: one of "global", "win", or "buf"
+/// - global_local: whether win or buf option has a global value
 ///
-///     - commalist: List of comma separated values
-///     - flaglist: List of single char flags
+/// - commalist: List of comma separated values
+/// - flaglist: List of single char flags
 ///
 /// When {scope} is not provided, the last set information applies to the local
 /// value in the current buffer or window if it is available, otherwise the

src/nvim/api/ui.c

@@ -165,7 +165,7 @@ void remote_ui_wait_for_attach(bool only_stdio)
 
 /// Activates UI events on the channel.
 ///
-/// Entry point of all UI clients.  Allows |\-\-embed| to continue startup.
+/// Entry point of all UI clients.  Allows |--embed| to continue startup.
 /// Implies that the client is ready to show the UI.  Adds the client to the
 /// list of UIs. |nvim_list_uis()|
 ///
@@ -541,7 +541,7 @@ void nvim_ui_pum_set_bounds(uint64_t channel_id, Float width, Float height, Floa
 ///
 /// @param channel_id
 /// @param event Event name
-/// @param payload Event payload
+/// @param value Event payload
 /// @param[out] err Error details, if any.
 void nvim_ui_term_event(uint64_t channel_id, String event, Object value, Error *err)
   FUNC_API_SINCE(12) FUNC_API_REMOTE_ONLY

src/nvim/api/vim.c

@@ -123,7 +123,7 @@ Dictionary nvim_get_hl(Integer ns_id, Dict(get_highlight) *opts, Arena *arena, E
 ///
 /// @note Unlike the `:highlight` command which can update a highlight group,
 ///       this function completely replaces the definition. For example:
-///       ``nvim_set_hl(0, 'Visual', {})`` will clear the highlight group
+///       `nvim_set_hl(0, 'Visual', {})` will clear the highlight group
 ///       'Visual'.
 ///
 /// @note The fg and bg keys also accept the string values `"fg"` or `"bg"`
@@ -549,6 +549,7 @@ ArrayOf(String) nvim_list_runtime_paths(Arena *arena, Error *err)
   return nvim_get_runtime_file(NULL_STRING, true, arena, err);
 }
 
+/// @nodoc
 Array nvim__runtime_inspect(Arena *arena)
 {
   return runtime_inspect(arena);
@@ -600,6 +601,7 @@ static bool find_runtime_cb(int num_fnames, char **fnames, bool all, void *c)
   return num_fnames > 0;
 }
 
+/// @nodoc
 String nvim__get_lib_dir(void)
   FUNC_API_RET_ALLOC
 {
@@ -1547,14 +1549,14 @@ Array nvim_get_api_info(uint64_t channel_id, Arena *arena)
 /// @param channel_id
 /// @param name Short name for the connected client
 /// @param version  Dictionary describing the version, with these
-///                 (optional) keys:
+///     (optional) keys:
 ///     - "major" major version (defaults to 0 if not set, for no release yet)
 ///     - "minor" minor version
 ///     - "patch" patch number
 ///     - "prerelease" string describing a prerelease, like "dev" or "beta1"
 ///     - "commit" hash or similar identifier of commit
 /// @param type Must be one of the following values. Client libraries should
-///             default to "remote" unless overridden by the user.
+///     default to "remote" unless overridden by the user.
 ///     - "remote" remote client connected "Nvim flavored" MessagePack-RPC (responses
 ///                must be in reverse order of requests). |msgpack-rpc|
 ///     - "msgpack-rpc" remote client connected to Nvim via fully MessagePack-RPC
@@ -1565,12 +1567,12 @@ Array nvim_get_api_info(uint64_t channel_id, Arena *arena)
 ///     - "host" plugin host, typically started by nvim
 ///     - "plugin" single plugin, started by nvim
 /// @param methods Builtin methods in the client. For a host, this does not
-///                include plugin methods which will be discovered later.
-///                The key should be the method name, the values are dicts with
-///                these (optional) keys (more keys may be added in future
-///                versions of Nvim, thus unknown keys are ignored. Clients
-///                must only use keys defined in this or later versions of
-///                Nvim):
+///     include plugin methods which will be discovered later.
+///     The key should be the method name, the values are dicts with
+///     these (optional) keys (more keys may be added in future
+///     versions of Nvim, thus unknown keys are ignored. Clients
+///     must only use keys defined in this or later versions of
+///     Nvim):
 ///     - "async"  if true, send as a notification. If false or unspecified,
 ///                use a blocking request
 ///     - "nargs" Number of arguments. Could be a single integer or an array
@@ -1979,7 +1981,7 @@ void nvim_select_popupmenu_item(Integer item, Boolean insert, Boolean finish, Di
   pum_ext_select_item((int)item, insert, finish);
 }
 
-/// NB: if your UI doesn't use hlstate, this will not return hlstate first time
+/// NB: if your UI doesn't use hlstate, this will not return hlstate first time.
 Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Arena *arena, Error *err)
 {
   Array ret = ARRAY_DICT_INIT;
@@ -2015,6 +2017,7 @@ Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Arena *arena, E
   return ret;
 }
 
+/// @nodoc
 void nvim__screenshot(String path)
   FUNC_API_FAST
 {
@@ -2029,6 +2032,7 @@ void nvim__invalidate_glyph_cache(void)
   must_redraw = UPD_CLEAR;
 }
 
+/// @nodoc
 Object nvim__unpack(String str, Arena *arena, Error *err)
   FUNC_API_FAST
 {
@@ -2319,6 +2323,7 @@ Dictionary nvim_eval_statusline(String str, Dict(eval_statusline) *opts, Arena *
   return result;
 }
 
+/// @nodoc
 void nvim_error_event(uint64_t channel_id, Integer lvl, String data)
   FUNC_API_REMOTE_ONLY
 {

src/nvim/api/vimscript.c

@@ -351,9 +351,7 @@ typedef struct {
   Object *ret_node_p;
 } ExprASTConvStackItem;
 
-/// @cond DOXYGEN_NOT_A_FUNCTION
 typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack;
-/// @endcond
 
 /// Parse a Vimscript expression.
 ///
@@ -387,8 +385,8 @@ typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack;
 ///          - "arg": String, error message argument.
 ///        - "len": Amount of bytes successfully parsed. With flags equal to ""
 ///                 that should be equal to the length of expr string.
-///                 (“Successfully parsed” here means “participated in AST
-///                  creation”, not “till the first error”.)
+///                 ("Successfully parsed" here means "participated in AST
+///                  creation", not "till the first error".)
 ///        - "ast": AST, either nil or a dictionary with these keys:
 ///          - "type": node type, one of the value names from ExprASTNodeType
 ///                    stringified without "kExprNode" prefix.

src/nvim/api/win_config.c

@@ -116,12 +116,12 @@
 ///   - width: Window width (in character cells). Minimum of 1.
 ///   - height: Window height (in character cells). Minimum of 1.
 ///   - bufpos: Places float relative to buffer text (only when
-///               relative="win"). Takes a tuple of zero-indexed [line, column].
-///               `row` and `col` if given are applied relative to this
-///               position, else they default to:
-///               - `row=1` and `col=0` if `anchor` is "NW" or "NE"
-///               - `row=0` and `col=0` if `anchor` is "SW" or "SE"
-///               (thus like a tooltip near the buffer text).
+///       relative="win"). Takes a tuple of zero-indexed [line, column].
+///       `row` and `col` if given are applied relative to this
+///       position, else they default to:
+///       - `row=1` and `col=0` if `anchor` is "NW" or "NE"
+///       - `row=0` and `col=0` if `anchor` is "SW" or "SE"
+///         (thus like a tooltip near the buffer text).
 ///   - row: Row position in units of "screen cell height", may be fractional.
 ///   - col: Column position in units of "screen cell width", may be
 ///            fractional.
@@ -153,7 +153,7 @@
 ///                    'fillchars' to a space char, and clearing the
 ///                    |hl-EndOfBuffer| region in 'winhighlight'.
 ///   - border: Style of (optional) window border. This can either be a string
-///      or an array. The string values are
+///     or an array. The string values are
 ///     - "none": No border (default).
 ///     - "single": A single line box.
 ///     - "double": A double line box.
@@ -161,21 +161,31 @@
 ///     - "solid": Adds padding by a single whitespace cell.
 ///     - "shadow": A drop shadow effect by blending with the background.
 ///     - If it is an array, it should have a length of eight or any divisor of
-///     eight. The array will specify the eight chars building up the border
-///     in a clockwise fashion starting with the top-left corner. As an
-///     example, the double box style could be specified as
+///       eight. The array will specify the eight chars building up the border
+///       in a clockwise fashion starting with the top-left corner. As an
+///       example, the double box style could be specified as:
+///       ```
 ///       [ "╔", "═" ,"╗", "║", "╝", "═", "╚", "║" ].
-///     If the number of chars are less than eight, they will be repeated. Thus
-///     an ASCII border could be specified as
+///       ```
+///       If the number of chars are less than eight, they will be repeated. Thus
+///       an ASCII border could be specified as
+///       ```
 ///       [ "/", "-", \"\\\\\", "|" ],
-///     or all chars the same as
+///       ```
+///       or all chars the same as
+///       ```
 ///       [ "x" ].
+///       ```
 ///     An empty string can be used to turn off a specific border, for instance,
+///     ```
 ///       [ "", "", "", ">", "", "", "", "<" ]
+///     ```
 ///     will only make vertical borders but not horizontal ones.
 ///     By default, `FloatBorder` highlight is used, which links to `WinSeparator`
 ///     when not defined.  It could also be specified by character:
+///     ```
 ///       [ ["+", "MyCorner"], ["x", "MyBorder"] ].
+///     ```
 ///   - title: Title (optional) in window border, string or list.
 ///     List should consist of `[text, highlight]` tuples.
 ///     If string, the default highlight group is `FloatTitle`.

src/nvim/api/window.c

@@ -452,6 +452,7 @@ Object nvim_win_call(Window window, LuaRef fun, Error *err)
 ///
 /// This takes precedence over the 'winhighlight' option.
 ///
+/// @param window
 /// @param ns_id the namespace to use
 /// @param[out] err Error details, if any
 void nvim_win_set_hl_ns(Window window, Integer ns_id, Error *err)