build: move all generator scripts to src/gen/

- Move all generator Lua scripts to the `src/gen/`
- Add a `.luarc.json` to `src/gen/`
- Add a `preload.lua` to `src/gen/`
  - Add `src` to `package.path` so it aligns with `.luarc.json'
- Fix all `require` statements in `src/gen/` so they are consistent:
    - `require('scripts.foo')` -> `require('gen.foo')`
    - `require('src.nvim.options')` -> `require('nvim.options')`
    - `require('api.dispatch_deprecated')` -> `require('nvim.api.dispatch_deprecated')`

src/gen/c_grammar.lua

@@ -0,0 +1,300 @@
+-- lpeg grammar for building api metadata from a set of header files. It
+-- ignores comments and preprocessor commands and parses a very small subset
+-- of C prototypes with a limited set of types
+
+--- @class nvim.c_grammar.Proto
+--- @field [1] 'proto'
+--- @field pos integer
+--- @field endpos integer
+--- @field fast boolean
+--- @field name string
+--- @field return_type string
+--- @field parameters [string, string][]
+--- @field static true?
+--- @field inline true?
+
+--- @class nvim.c_grammar.Preproc
+--- @field [1] 'preproc'
+--- @field content string
+
+--- @class nvim.c_grammar.Empty
+--- @field [1] 'empty'
+
+--- @alias nvim.c_grammar.result
+--- | nvim.c_grammar.Proto
+--- | nvim.c_grammar.Preproc
+--- | nvim.c_grammar.Empty
+
+--- @class nvim.c_grammar
+--- @field match fun(self, input: string): nvim.c_grammar.result[]
+
+local lpeg = vim.lpeg
+
+local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V
+local C, Ct, Cc, Cg, Cp = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cp
+
+--- @param pat vim.lpeg.Pattern
+local function rep(pat)
+  return pat ^ 0
+end
+
+--- @param pat vim.lpeg.Pattern
+local function rep1(pat)
+  return pat ^ 1
+end
+
+--- @param pat vim.lpeg.Pattern
+local function opt(pat)
+  return pat ^ -1
+end
+
+local any = P(1)
+local letter = R('az', 'AZ') + S('_$')
+local num = R('09')
+local alpha = letter + num
+local nl = P('\r\n') + P('\n')
+local space = S(' \t')
+local str = P('"') * rep((P('\\') * any) + (1 - P('"'))) * P('"')
+local char = P("'") * (any - P("'")) * P("'")
+local ws = space + nl
+local wb = #-alpha -- word boundary
+local id = letter * rep(alpha)
+
+local comment_inline = P('/*') * rep(1 - P('*/')) * P('*/')
+local comment = P('//') * rep(1 - nl) * nl
+local preproc = Ct(Cc('preproc') * P('#') * Cg(rep(1 - nl) * nl, 'content'))
+
+local fill = rep(ws + comment_inline + comment + preproc)
+
+--- @param s string
+--- @return vim.lpeg.Pattern
+local function word(s)
+  return fill * P(s) * wb * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function comma1(x)
+  return x * rep(fill * P(',') * fill * x)
+end
+
+--- @param v string
+local function Pf(v)
+  return fill * P(v) * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function paren(x)
+  return P('(') * fill * x * fill * P(')')
+end
+
+local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(1 - nl), 'comment')))
+
+local braces = P({
+  'S',
+  A = comment_inline + comment + preproc + str + char + (any - S('{}')),
+  S = P('{') * rep(V('A')) * rep(V('S') + V('A')) * P('}'),
+})
+
+-- stylua: ignore start
+local typed_container = P({
+  'S',
+  S = (
+    (P('Union') * paren(comma1(V('ID'))))
+    + (P('ArrayOf') * paren(id * opt(P(',') * fill * rep1(num))))
+    + (P('DictOf') * paren(id))
+    + (P('LuaRefOf') * paren(
+      paren(comma1((V('ID') + str) * rep1(ws) * opt(P('*')) * id))
+      * opt(P(',') * fill * opt(P('*')) * V('ID'))
+    ))
+    + (P('Dict') * paren(id))),
+  ID = V('S') + id,
+})
+-- stylua: ignore end
+
+local ptr_mod = word('restrict') + word('__restrict') + word('const')
+local opt_ptr = rep(Pf('*') * opt(ptr_mod))
+
+--- @param name string
+--- @param var string
+--- @return vim.lpeg.Pattern
+local function attr(name, var)
+  return Cg((P(name) * Cc(true)), var)
+end
+
+--- @param name string
+--- @param var string
+--- @return vim.lpeg.Pattern
+local function attr_num(name, var)
+  return Cg((P(name) * paren(C(rep1(num)))), var)
+end
+
+local fattr = (
+  attr_num('FUNC_API_SINCE', 'since')
+  + attr_num('FUNC_API_DEPRECATED_SINCE', 'deprecated_since')
+  + attr('FUNC_API_FAST', 'fast')
+  + attr('FUNC_API_RET_ALLOC', 'ret_alloc')
+  + attr('FUNC_API_NOEXPORT', 'noexport')
+  + attr('FUNC_API_REMOTE_ONLY', 'remote_only')
+  + attr('FUNC_API_LUA_ONLY', 'lua_only')
+  + attr('FUNC_API_TEXTLOCK_ALLOW_CMDWIN', 'textlock_allow_cmdwin')
+  + attr('FUNC_API_TEXTLOCK', 'textlock')
+  + attr('FUNC_API_REMOTE_IMPL', 'remote_impl')
+  + attr('FUNC_API_COMPOSITOR_IMPL', 'compositor_impl')
+  + attr('FUNC_API_CLIENT_IMPL', 'client_impl')
+  + attr('FUNC_API_CLIENT_IGNORE', 'client_ignore')
+  + (P('FUNC_') * rep(alpha) * opt(fill * paren(rep(1 - P(')') * any))))
+)
+
+local void = P('void') * wb
+
+local api_param_type = (
+  (word('Error') * opt_ptr * Cc('error'))
+  + (word('Arena') * opt_ptr * Cc('arena'))
+  + (word('lua_State') * opt_ptr * Cc('lstate'))
+)
+
+local ctype = C(
+  opt(word('const'))
+    * (
+      typed_container
+      -- 'unsigned' is a type modifier, and a type itself
+      + (word('unsigned char') + word('unsigned'))
+      + (word('struct') * fill * id)
+      + id
+    )
+    * opt(word('const'))
+    * opt_ptr
+)
+
+local return_type = (C(void) * fill) + ctype
+
+-- stylua: ignore start
+local params = Ct(
+  (void * #P(')'))
+  + comma1(Ct(
+      (api_param_type + ctype)
+      * fill
+      * C(id)
+      * rep(Pf('[') * rep(alpha) * Pf(']'))
+      * rep(fill * fattr)
+    ))
+    * opt(Pf(',') * P('...'))
+)
+-- stylua: ignore end
+
+local ignore_line = rep1(1 - nl) * nl
+local empty_line = Ct(Cc('empty') * nl * nl)
+
+local proto_name = opt_ptr * fill * id
+
+-- __inline is used in MSVC
+local decl_mod = (
+  Cg(word('static') * Cc(true), 'static')
+  + Cg((word('inline') + word('__inline')) * Cc(true), 'inline')
+)
+
+local proto = Ct(
+  Cg(Cp(), 'pos')
+    * Cc('proto')
+    * -#P('typedef')
+    * #alpha
+    * opt(P('DLLEXPORT') * rep1(ws))
+    * rep(decl_mod)
+    * Cg(return_type, 'return_type')
+    * fill
+    * Cg(proto_name, 'name')
+    * fill
+    * paren(Cg(params, 'parameters'))
+    * Cg(Cc(false), 'fast')
+    * rep(fill * fattr)
+    * Cg(Cp(), 'endpos')
+    * (fill * (S(';') + braces))
+)
+
+local keyset_field = Ct(
+  Cg(ctype, 'type')
+    * fill
+    * Cg(id, 'name')
+    * fill
+    * opt(P('DictKey') * paren(Cg(rep1(1 - P(')')), 'dict_key')))
+    * Pf(';')
+)
+
+local keyset = Ct(
+  P('typedef')
+    * word('struct')
+    * Pf('{')
+    * Cg(Ct(rep1(keyset_field)), 'fields')
+    * Pf('}')
+    * P('Dict')
+    * paren(Cg(id, 'keyset_name'))
+    * Pf(';')
+)
+
+local grammar =
+  Ct(rep1(empty_line + proto + cdoc_comment + comment + preproc + ws + keyset + ignore_line))
+
+if arg[1] == '--test' then
+  for i, t in ipairs({
+    'void multiqueue_put_event(MultiQueue *self, Event event) {} ',
+    'void *xmalloc(size_t size) {} ',
+    {
+      'struct tm *os_localtime_r(const time_t *restrict clock,',
+      '                          struct tm *restrict result) FUNC_ATTR_NONNULL_ALL {}',
+    },
+    {
+      '_Bool',
+      '# 163 "src/nvim/event/multiqueue.c"',
+      '    multiqueue_empty(MultiQueue *self)',
+      '{}',
+    },
+    'const char *find_option_end(const char *arg, OptIndex *opt_idxp) {}',
+    'bool semsg(const char *const fmt, ...) {}',
+    'int32_t utf_ptr2CharInfo_impl(uint8_t const *p, uintptr_t const len) {}',
+    'void ex_argdedupe(exarg_T *eap FUNC_ATTR_UNUSED) {}',
+    'static TermKeySym register_c0(TermKey *tk, TermKeySym sym, unsigned char ctrl, const char *name) {}',
+    'unsigned get_bkc_flags(buf_T *buf) {}',
+    'char *xstpcpy(char *restrict dst, const char *restrict src) {}',
+    'bool try_leave(const TryState *const tstate, Error *const err) {}',
+    'void api_set_error(ErrorType errType) {}',
+    {
+      'void nvim_subscribe(uint64_t channel_id, String event)',
+      'FUNC_API_SINCE(1) FUNC_API_DEPRECATED_SINCE(13) FUNC_API_REMOTE_ONLY',
+      '{}',
+    },
+
+    -- Do not consume leading preproc statements
+    {
+      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
+      'static __inline int mark_global_index(const char name)',
+      '  FUNC_ATTR_CONST',
+      '{}',
+    },
+    {
+      '',
+      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
+      'static __inline int mark_global_index(const char name)',
+      '{}',
+    },
+    {
+      'size_t xstrlcpy(char *__restrict dst, const char *__restrict src, size_t dsize)',
+      ' FUNC_ATTR_NONNULL_ALL',
+      ' {}',
+    },
+  }) do
+    if type(t) == 'table' then
+      t = table.concat(t, '\n') .. '\n'
+    end
+    t = t:gsub(' +', ' ')
+    local r = grammar:match(t)
+    if not r then
+      print('Test ' .. i .. ' failed')
+      print('    |' .. table.concat(vim.split(t, '\n'), '\n    |'))
+    end
+  end
+end
+
+return {
+  grammar = grammar --[[@as nvim.c_grammar]],
+  typed_container = typed_container,
+}

src/gen/cdoc_grammar.lua

@@ -0,0 +1,87 @@
+--[[!
+LPEG grammar for C doc comments
+]]
+
+--- @class nvim.cdoc.Param
+--- @field kind 'param'
+--- @field name string
+--- @field desc? string
+
+--- @class nvim.cdoc.Return
+--- @field kind 'return'
+--- @field desc string
+
+--- @class nvim.cdoc.Note
+--- @field desc? string
+
+--- @alias nvim.cdoc.grammar.result
+--- | nvim.cdoc.Param
+--- | nvim.cdoc.Return
+--- | nvim.cdoc.Note
+
+--- @class nvim.cdoc.grammar
+--- @field match fun(self, input: string): nvim.cdoc.grammar.result?
+
+local lpeg = vim.lpeg
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local Ct, Cg = lpeg.Ct, lpeg.Cg
+
+--- @param x vim.lpeg.Pattern
+local function rep(x)
+  return x ^ 0
+end
+
+--- @param x vim.lpeg.Pattern
+local function rep1(x)
+  return x ^ 1
+end
+
+--- @param x vim.lpeg.Pattern
+local function opt(x)
+  return x ^ -1
+end
+
+local nl = P('\r\n') + P('\n')
+local ws = rep1(S(' \t') + nl)
+
+local any = P(1) -- (consume one character)
+local letter = R('az', 'AZ') + S('_$')
+local ident = letter * rep(letter + R('09'))
+
+local io = P('[') * (P('in') + P('out') + P('inout')) * P(']')
+
+--- @param x string
+local function Pf(x)
+  return opt(ws) * P(x) * opt(ws)
+end
+
+--- @type table<string,vim.lpeg.Pattern>
+local v = setmetatable({}, {
+  __index = function(_, k)
+    return lpeg.V(k)
+  end,
+})
+
+local grammar = P {
+  rep1(P('@') * v.ats),
+
+  ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc,
+
+  at_param = Ct(
+    Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc'))
+  ),
+
+  at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))),
+
+  at_deprecated = Ct(Cg(P('deprecated'), 'kind')),
+
+  at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')),
+
+  at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')),
+
+  at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')),
+
+  at_nodoc = Ct(Cg(P('nodoc'), 'kind')),
+}
+
+return grammar --[[@as nvim.cdoc.grammar]]

src/gen/cdoc_parser.lua

@@ -0,0 +1,223 @@
+local cdoc_grammar = require('gen.cdoc_grammar')
+local c_grammar = require('gen.c_grammar')
+
+--- @class nvim.cdoc.parser.param
+--- @field name string
+--- @field type string
+--- @field desc string
+
+--- @class nvim.cdoc.parser.return
+--- @field name string
+--- @field type string
+--- @field desc string
+
+--- @class nvim.cdoc.parser.note
+--- @field desc string
+
+--- @class nvim.cdoc.parser.brief
+--- @field kind 'brief'
+--- @field desc string
+
+--- @class nvim.cdoc.parser.fun
+--- @field name string
+--- @field params nvim.cdoc.parser.param[]
+--- @field returns nvim.cdoc.parser.return[]
+--- @field desc string
+--- @field deprecated? true
+--- @field since? string
+--- @field attrs? string[]
+--- @field nodoc? true
+--- @field notes? nvim.cdoc.parser.note[]
+--- @field see? nvim.cdoc.parser.note[]
+
+--- @class nvim.cdoc.parser.State
+--- @field doc_lines? string[]
+--- @field cur_obj? nvim.cdoc.parser.obj
+--- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note
+--- @field last_doc_item_indent? integer
+
+--- @alias nvim.cdoc.parser.obj
+--- | nvim.cdoc.parser.fun
+--- | nvim.cdoc.parser.brief
+
+--- If we collected any `---` lines. Add them to the existing (or new) object
+--- Used for function/class descriptions and multiline param descriptions.
+--- @param state nvim.cdoc.parser.State
+local function add_doc_lines_to_obj(state)
+  if state.doc_lines then
+    state.cur_obj = state.cur_obj or {}
+    local cur_obj = assert(state.cur_obj)
+    local txt = table.concat(state.doc_lines, '\n')
+    if cur_obj.desc then
+      cur_obj.desc = cur_obj.desc .. '\n' .. txt
+    else
+      cur_obj.desc = txt
+    end
+    state.doc_lines = nil
+  end
+end
+
+--- @param line string
+--- @param state nvim.cdoc.parser.State
+local function process_doc_line(line, state)
+  line = line:gsub('^%s+@', '@')
+
+  local parsed = cdoc_grammar:match(line)
+
+  if not parsed then
+    if line:match('^ ') then
+      line = line:sub(2)
+    end
+
+    if state.last_doc_item then
+      if not state.last_doc_item_indent then
+        state.last_doc_item_indent = #line:match('^%s*') + 1
+      end
+      state.last_doc_item.desc = (state.last_doc_item.desc or '')
+        .. '\n'
+        .. line:sub(state.last_doc_item_indent or 1)
+    else
+      state.doc_lines = state.doc_lines or {}
+      table.insert(state.doc_lines, line)
+    end
+    return
+  end
+
+  state.last_doc_item_indent = nil
+  state.last_doc_item = nil
+
+  local kind = parsed.kind
+
+  state.cur_obj = state.cur_obj or {}
+  local cur_obj = assert(state.cur_obj)
+
+  if kind == 'brief' then
+    state.cur_obj = {
+      kind = 'brief',
+      desc = parsed.desc,
+    }
+  elseif kind == 'param' then
+    state.last_doc_item_indent = nil
+    cur_obj.params = cur_obj.params or {}
+    state.last_doc_item = {
+      name = parsed.name,
+      desc = parsed.desc,
+    }
+    table.insert(cur_obj.params, state.last_doc_item)
+  elseif kind == 'return' then
+    cur_obj.returns = { {
+      desc = parsed.desc,
+    } }
+    state.last_doc_item_indent = nil
+    state.last_doc_item = cur_obj.returns[1]
+  elseif kind == 'deprecated' then
+    cur_obj.deprecated = true
+  elseif kind == 'nodoc' then
+    cur_obj.nodoc = true
+  elseif kind == 'since' then
+    cur_obj.since = parsed.desc
+  elseif kind == 'see' then
+    cur_obj.see = cur_obj.see or {}
+    table.insert(cur_obj.see, { desc = parsed.desc })
+  elseif kind == 'note' then
+    state.last_doc_item_indent = nil
+    state.last_doc_item = {
+      desc = parsed.desc,
+    }
+    cur_obj.notes = cur_obj.notes or {}
+    table.insert(cur_obj.notes, state.last_doc_item)
+  else
+    error('Unhandled' .. vim.inspect(parsed))
+  end
+end
+
+--- @param item table
+--- @param state nvim.cdoc.parser.State
+local function process_proto(item, state)
+  state.cur_obj = state.cur_obj or {}
+  local cur_obj = assert(state.cur_obj)
+  cur_obj.name = item.name
+  cur_obj.params = cur_obj.params or {}
+
+  for _, p in ipairs(item.parameters) do
+    local param = { name = p[2], type = p[1] }
+    local added = false
+    for _, cp in ipairs(cur_obj.params) do
+      if cp.name == param.name then
+        cp.type = param.type
+        added = true
+        break
+      end
+    end
+
+    if not added then
+      table.insert(cur_obj.params, param)
+    end
+  end
+
+  cur_obj.returns = cur_obj.returns or { {} }
+  cur_obj.returns[1].type = item.return_type
+
+  for _, a in ipairs({
+    'fast',
+    'remote_only',
+    'lua_only',
+    'textlock',
+    'textlock_allow_cmdwin',
+  }) do
+    if item[a] then
+      cur_obj.attrs = cur_obj.attrs or {}
+      table.insert(cur_obj.attrs, a)
+    end
+  end
+
+  cur_obj.deprecated_since = item.deprecated_since
+
+  -- Remove some arguments
+  for i = #cur_obj.params, 1, -1 do
+    local p = cur_obj.params[i]
+    if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then
+      table.remove(cur_obj.params, i)
+    end
+  end
+end
+
+local M = {}
+
+--- @param filename string
+--- @return {} classes
+--- @return nvim.cdoc.parser.fun[] funs
+--- @return string[] briefs
+function M.parse(filename)
+  local funs = {} --- @type nvim.cdoc.parser.fun[]
+  local briefs = {} --- @type string[]
+  local state = {} --- @type nvim.cdoc.parser.State
+
+  local txt = assert(io.open(filename, 'r')):read('*all')
+
+  local parsed = c_grammar.grammar:match(txt)
+  for _, item in ipairs(parsed) do
+    if item.comment then
+      process_doc_line(item.comment, state)
+    else
+      add_doc_lines_to_obj(state)
+      if item[1] == 'proto' then
+        process_proto(item, state)
+        table.insert(funs, state.cur_obj)
+      end
+      local cur_obj = state.cur_obj
+      if cur_obj and not item.static then
+        if cur_obj.kind == 'brief' then
+          table.insert(briefs, cur_obj.desc)
+        end
+      end
+      state = {}
+    end
+  end
+
+  return {}, funs, briefs
+end
+
+-- M.parse('src/nvim/api/vim.c')
+
+return M

src/gen/dump_bin_array.lua

@@ -0,0 +1,17 @@
+local function dump_bin_array(output, name, data)
+  output:write([[
+  static const uint8_t ]] .. name .. [[[] = {
+]])
+
+  for i = 1, #data do
+    output:write(string.byte(data, i) .. ', ')
+    if i % 10 == 0 then
+      output:write('\n  ')
+    end
+  end
+  output:write([[
+};
+]])
+end
+
+return dump_bin_array

src/gen/gen_api_dispatch.lua

@@ -0,0 +1,990 @@
+-- Example (manual) invocation:
+--
+--    make
+--    cp build/nvim_version.lua src/nvim/
+--    cd src/nvim
+--    nvim -l generators/gen_api_dispatch.lua "../../build/src/nvim/auto/api/private/dispatch_wrappers.generated.h" "../../build/src/nvim/auto/api/private/api_metadata.generated.h" "../../build/funcs_metadata.mpack" "../../build/src/nvim/auto/lua_api_c_bindings.generated.h" "../../build/src/nvim/auto/keysets_defs.generated.h" "../../build/ui_metadata.mpack" "../../build/cmake.config/auto/versiondef_git.h" "./api/autocmd.h" "./api/buffer.h" "./api/command.h" "./api/deprecated.h" "./api/extmark.h" "./api/keysets_defs.h" "./api/options.h" "./api/tabpage.h" "./api/ui.h" "./api/vim.h" "./api/vimscript.h" "./api/win_config.h" "./api/window.h" "../../build/include/api/autocmd.h.generated.h" "../../build/include/api/buffer.h.generated.h" "../../build/include/api/command.h.generated.h" "../../build/include/api/deprecated.h.generated.h" "../../build/include/api/extmark.h.generated.h" "../../build/include/api/options.h.generated.h" "../../build/include/api/tabpage.h.generated.h" "../../build/include/api/ui.h.generated.h" "../../build/include/api/vim.h.generated.h" "../../build/include/api/vimscript.h.generated.h" "../../build/include/api/win_config.h.generated.h" "../../build/include/api/window.h.generated.h"
+
+local mpack = vim.mpack
+
+local hashy = require 'gen.hashy'
+
+local pre_args = 7
+assert(#arg >= pre_args)
+-- output h file with generated dispatch functions (dispatch_wrappers.generated.h)
+local dispatch_outputf = arg[1]
+-- output h file with packed metadata (api_metadata.generated.h)
+local api_metadata_outputf = arg[2]
+-- output metadata mpack file, for use by other build scripts (funcs_metadata.mpack)
+local mpack_outputf = arg[3]
+local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c
+local keysets_outputf = arg[5] -- keysets_defs.generated.h
+local ui_metadata_inputf = arg[6] -- ui events metadata
+local git_version_inputf = arg[7] -- git version header
+
+local functions = {}
+
+-- names of all headers relative to the source root (for inclusion in the
+-- generated file)
+local headers = {}
+
+-- set of function names, used to detect duplicates
+local function_names = {}
+
+local c_grammar = require('gen.c_grammar')
+
+local startswith = vim.startswith
+
+local function add_function(fn)
+  local public = startswith(fn.name, 'nvim_') or fn.deprecated_since
+  if public and not fn.noexport then
+    functions[#functions + 1] = fn
+    function_names[fn.name] = true
+    if
+      #fn.parameters >= 2
+      and fn.parameters[2][1] == 'Array'
+      and fn.parameters[2][2] == 'uidata'
+    then
+      -- function receives the "args" as a parameter
+      fn.receives_array_args = true
+      -- remove the args parameter
+      table.remove(fn.parameters, 2)
+    end
+    if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
+      -- this function should receive the channel id
+      fn.receives_channel_id = true
+      -- remove the parameter since it won't be passed by the api client
+      table.remove(fn.parameters, 1)
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then
+      -- function can fail if the last parameter type is 'Error'
+      fn.can_fail = true
+      -- remove the error parameter, msgpack has it's own special field
+      -- for specifying errors
+      fn.parameters[#fn.parameters] = nil
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'lstate' then
+      fn.has_lua_imp = true
+      fn.parameters[#fn.parameters] = nil
+    end
+    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'arena' then
+      fn.receives_arena = true
+      fn.parameters[#fn.parameters] = nil
+    end
+  end
+end
+
+local keysets = {}
+
+local function add_keyset(val)
+  local keys = {}
+  local types = {}
+  local c_names = {}
+  local is_set_name = 'is_set__' .. val.keyset_name .. '_'
+  local has_optional = false
+  for i, field in ipairs(val.fields) do
+    local dict_key = field.dict_key or field.name
+    if field.type ~= 'Object' then
+      types[dict_key] = field.type
+    end
+    if field.name ~= is_set_name and field.type ~= 'OptionalKeys' then
+      table.insert(keys, dict_key)
+      if dict_key ~= field.name then
+        c_names[dict_key] = field.name
+      end
+    else
+      if i > 1 then
+        error("'is_set__{type}_' must be first if present")
+      elseif field.name ~= is_set_name then
+        error(val.keyset_name .. ': name of first key should be ' .. is_set_name)
+      elseif field.type ~= 'OptionalKeys' then
+        error("'" .. is_set_name .. "' must have type 'OptionalKeys'")
+      end
+      has_optional = true
+    end
+  end
+  table.insert(keysets, {
+    name = val.keyset_name,
+    keys = keys,
+    c_names = c_names,
+    types = types,
+    has_optional = has_optional,
+  })
+end
+
+local ui_options_text = nil
+
+-- read each input file, parse and append to the api metadata
+for i = pre_args + 1, #arg do
+  local full_path = arg[i]
+  local parts = {}
+  for part in string.gmatch(full_path, '[^/]+') do
+    parts[#parts + 1] = part
+  end
+  headers[#headers + 1] = parts[#parts - 1] .. '/' .. parts[#parts]
+
+  local input = assert(io.open(full_path, 'rb'))
+
+  local text = input:read('*all')
+  local tmp = c_grammar.grammar:match(text)
+  for j = 1, #tmp do
+    local val = tmp[j]
+    if val.keyset_name then
+      add_keyset(val)
+    elseif val.name then
+      add_function(val)
+    end
+  end
+
+  ui_options_text = ui_options_text or string.match(text, 'ui_ext_names%[][^{]+{([^}]+)}')
+  input:close()
+end
+
+local function shallowcopy(orig)
+  local copy = {}
+  for orig_key, orig_value in pairs(orig) do
+    copy[orig_key] = orig_value
+  end
+  return copy
+end
+
+-- Export functions under older deprecated names.
+-- These will be removed eventually.
+local deprecated_aliases = require('nvim.api.dispatch_deprecated')
+for _, f in ipairs(shallowcopy(functions)) do
+  local ismethod = false
+  if startswith(f.name, 'nvim_') then
+    if startswith(f.name, 'nvim__') or f.name == 'nvim_error_event' then
+      f.since = -1
+    elseif f.since == nil then
+      print('Function ' .. f.name .. ' lacks since field.\n')
+      os.exit(1)
+    end
+    f.since = tonumber(f.since)
+    if f.deprecated_since ~= nil then
+      f.deprecated_since = tonumber(f.deprecated_since)
+    end
+
+    if startswith(f.name, 'nvim_buf_') then
+      ismethod = true
+    elseif startswith(f.name, 'nvim_win_') then
+      ismethod = true
+    elseif startswith(f.name, 'nvim_tabpage_') then
+      ismethod = true
+    end
+    f.remote = f.remote_only or not f.lua_only
+    f.lua = f.lua_only or not f.remote_only
+    f.eval = (not f.lua_only) and not f.remote_only
+  else
+    f.deprecated_since = tonumber(f.deprecated_since)
+    assert(f.deprecated_since == 1)
+    f.remote = true
+    f.since = 0
+  end
+  f.method = ismethod
+  local newname = deprecated_aliases[f.name]
+  if newname ~= nil then
+    if function_names[newname] then
+      -- duplicate
+      print(
+        'Function '
+          .. f.name
+          .. ' has deprecated alias\n'
+          .. newname
+          .. ' which has a separate implementation.\n'
+          .. 'Please remove it from src/nvim/api/dispatch_deprecated.lua'
+      )
+      os.exit(1)
+    end
+    local newf = shallowcopy(f)
+    newf.name = newname
+    if newname == 'ui_try_resize' then
+      -- The return type was incorrectly set to Object in 0.1.5.
+      -- Keep it that way for clients that rely on this.
+      newf.return_type = 'Object'
+    end
+    newf.impl_name = f.name
+    newf.lua = false
+    newf.eval = false
+    newf.since = 0
+    newf.deprecated_since = 1
+    functions[#functions + 1] = newf
+  end
+end
+
+-- don't expose internal attributes like "impl_name" in public metadata
+local exported_attributes = { 'name', 'return_type', 'method', 'since', 'deprecated_since' }
+local exported_functions = {}
+for _, f in ipairs(functions) do
+  if not (startswith(f.name, 'nvim__') or f.name == 'nvim_error_event' or f.name == 'redraw') then
+    local f_exported = {}
+    for _, attr in ipairs(exported_attributes) do
+      f_exported[attr] = f[attr]
+    end
+    f_exported.parameters = {}
+    for i, param in ipairs(f.parameters) do
+      if param[1] == 'DictOf(LuaRef)' then
+        param = { 'Dict', param[2] }
+      elseif startswith(param[1], 'Dict(') then
+        param = { 'Dict', param[2] }
+      end
+      f_exported.parameters[i] = param
+    end
+    if startswith(f.return_type, 'Dict(') then
+      f_exported.return_type = 'Dict'
+    end
+    exported_functions[#exported_functions + 1] = f_exported
+  end
+end
+
+local ui_options = { 'rgb' }
+for x in string.gmatch(ui_options_text, '"([a-z][a-z_]+)"') do
+  table.insert(ui_options, x)
+end
+
+local version = require 'nvim_version' -- `build/nvim_version.lua` file.
+local git_version = io.open(git_version_inputf):read '*a'
+local version_build = string.match(git_version, '#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
+
+-- serialize the API metadata using msgpack and embed into the resulting
+-- binary for easy querying by clients
+local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb'))
+local pieces = {}
+
+-- Naively using mpack.encode({foo=x, bar=y}) will make the build
+-- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead
+local function fixdict(num)
+  if num > 15 then
+    error 'implement more dict codes'
+  end
+  table.insert(pieces, string.char(128 + num))
+end
+local function put(item, item2)
+  table.insert(pieces, mpack.encode(item))
+  if item2 ~= nil then
+    table.insert(pieces, mpack.encode(item2))
+  end
+end
+
+fixdict(6)
+
+put('version')
+fixdict(1 + #version)
+for _, item in ipairs(version) do
+  -- NB: all items are mandatory. But any error will be less confusing
+  -- with placeholder vim.NIL (than invalid mpack data)
+  local val = item[2] == nil and vim.NIL or item[2]
+  put(item[1], val)
+end
+put('build', version_build)
+
+put('functions', exported_functions)
+put('ui_events')
+table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all'))
+put('ui_options', ui_options)
+
+put('error_types')
+fixdict(2)
+put('Exception', { id = 0 })
+put('Validation', { id = 1 })
+
+put('types')
+local types =
+  { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } }
+fixdict(#types)
+for i, item in ipairs(types) do
+  put(item[1])
+  fixdict(2)
+  put('id', i - 1)
+  put('prefix', item[2])
+end
+
+local packed = table.concat(pieces)
+local dump_bin_array = require('gen.dump_bin_array')
+dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
+api_metadata_output:close()
+
+-- start building the dispatch wrapper output
+local output = assert(io.open(dispatch_outputf, 'wb'))
+
+local keysets_defs = assert(io.open(keysets_outputf, 'wb'))
+
+-- ===========================================================================
+-- NEW API FILES MUST GO HERE.
+--
+--  When creating a new API file, you must include it here,
+--  so that the dispatcher can find the C functions that you are creating!
+-- ===========================================================================
+output:write([[
+#include "nvim/errors.h"
+#include "nvim/ex_docmd.h"
+#include "nvim/ex_getln.h"
+#include "nvim/globals.h"
+#include "nvim/log.h"
+#include "nvim/map_defs.h"
+
+#include "nvim/api/autocmd.h"
+#include "nvim/api/buffer.h"
+#include "nvim/api/command.h"
+#include "nvim/api/deprecated.h"
+#include "nvim/api/extmark.h"
+#include "nvim/api/options.h"
+#include "nvim/api/tabpage.h"
+#include "nvim/api/ui.h"
+#include "nvim/api/vim.h"
+#include "nvim/api/vimscript.h"
+#include "nvim/api/win_config.h"
+#include "nvim/api/window.h"
+#include "nvim/ui_client.h"
+
+]])
+
+keysets_defs:write('// IWYU pragma: private, include "nvim/api/private/dispatch.h"\n\n')
+
+for _, k in ipairs(keysets) do
+  local neworder, hashfun = hashy.hashy_hash(k.name, k.keys, function(idx)
+    return k.name .. '_table[' .. idx .. '].str'
+  end)
+
+  keysets_defs:write('extern KeySetLink ' .. k.name .. '_table[' .. (1 + #neworder) .. '];\n')
+
+  local function typename(type)
+    if type == 'HLGroupID' then
+      return 'kObjectTypeInteger'
+    elseif not type or vim.startswith(type, 'Union') then
+      return 'kObjectTypeNil'
+    elseif vim.startswith(type, 'LuaRefOf') then
+      return 'kObjectTypeLuaRef'
+    elseif type == 'StringArray' then
+      return 'kUnpackTypeStringArray'
+    elseif vim.startswith(type, 'ArrayOf') then
+      return 'kObjectTypeArray'
+    else
+      return 'kObjectType' .. type
+    end
+  end
+
+  output:write('KeySetLink ' .. k.name .. '_table[] = {\n')
+  for i, key in ipairs(neworder) do
+    local ind = -1
+    if k.has_optional then
+      ind = i
+      keysets_defs:write('#define KEYSET_OPTIDX_' .. k.name .. '__' .. key .. ' ' .. ind .. '\n')
+    end
+    output:write(
+      '  {"'
+        .. key
+        .. '", offsetof(KeyDict_'
+        .. k.name
+        .. ', '
+        .. (k.c_names[key] or key)
+        .. '), '
+        .. typename(k.types[key])
+        .. ', '
+        .. ind
+        .. ', '
+        .. (k.types[key] == 'HLGroupID' and 'true' or 'false')
+        .. '},\n'
+    )
+  end
+  output:write('  {NULL, 0, kObjectTypeNil, -1, false},\n')
+  output:write('};\n\n')
+
+  output:write(hashfun)
+
+  output:write([[
+KeySetLink *KeyDict_]] .. k.name .. [[_get_field(const char *str, size_t len)
+{
+  int hash = ]] .. k.name .. [[_hash(str, len);
+  if (hash == -1) {
+    return NULL;
+  }
+  return &]] .. k.name .. [[_table[hash];
+}
+
+]])
+end
+
+local function real_type(type)
+  local rv = type
+  local rmatch = string.match(type, 'Dict%(([_%w]+)%)')
+  if rmatch then
+    return 'KeyDict_' .. rmatch
+  elseif c_grammar.typed_container:match(rv) then
+    if rv:match('Array') then
+      rv = 'Array'
+    else
+      rv = 'Dict'
+    end
+  end
+  return rv
+end
+
+local function attr_name(rt)
+  if rt == 'Float' then
+    return 'floating'
+  else
+    return rt:lower()
+  end
+end
+
+-- start the handler functions. Visit each function metadata to build the
+-- handler function with code generated for validating arguments and calling to
+-- the real API.
+for i = 1, #functions do
+  local fn = functions[i]
+  if fn.impl_name == nil and fn.remote then
+    local args = {}
+
+    output:write(
+      'Object handle_' .. fn.name .. '(uint64_t channel_id, Array args, Arena* arena, Error *error)'
+    )
+    output:write('\n{')
+    output:write('\n#ifdef NVIM_LOG_DEBUG')
+    output:write('\n  DLOG("RPC: ch %" PRIu64 ": invoke ' .. fn.name .. '", channel_id);')
+    output:write('\n#endif')
+    output:write('\n  Object ret = NIL;')
+    -- Declare/initialize variables that will hold converted arguments
+    for j = 1, #fn.parameters do
+      local param = fn.parameters[j]
+      local rt = real_type(param[1])
+      local converted = 'arg_' .. j
+      output:write('\n  ' .. rt .. ' ' .. converted .. ';')
+    end
+    output:write('\n')
+    if not fn.receives_array_args then
+      output:write('\n  if (args.size != ' .. #fn.parameters .. ') {')
+      output:write(
+        '\n    api_set_error(error, kErrorTypeException, \
+        "Wrong number of arguments: expecting '
+          .. #fn.parameters
+          .. ' but got %zu", args.size);'
+      )
+      output:write('\n    goto cleanup;')
+      output:write('\n  }\n')
+    end
+
+    -- Validation/conversion for each argument
+    for j = 1, #fn.parameters do
+      local converted, param
+      param = fn.parameters[j]
+      converted = 'arg_' .. j
+      local rt = real_type(param[1])
+      if rt == 'Object' then
+        output:write('\n  ' .. converted .. ' = args.items[' .. (j - 1) .. '];\n')
+      elseif rt:match('^KeyDict_') then
+        converted = '&' .. converted
+        output:write('\n  if (args.items[' .. (j - 1) .. '].type == kObjectTypeDict) {') --luacheck: ignore 631
+        output:write('\n    memset(' .. converted .. ', 0, sizeof(*' .. converted .. '));') -- TODO: neeeee
+        output:write(
+          '\n    if (!api_dict_to_keydict('
+            .. converted
+            .. ', '
+            .. rt
+            .. '_get_field, args.items['
+            .. (j - 1)
+            .. '].data.dict, error)) {'
+        )
+        output:write('\n      goto cleanup;')
+        output:write('\n    }')
+        output:write(
+          '\n  } else if (args.items['
+            .. (j - 1)
+            .. '].type == kObjectTypeArray && args.items['
+            .. (j - 1)
+            .. '].data.array.size == 0) {'
+        ) --luacheck: ignore 631
+        output:write('\n    memset(' .. converted .. ', 0, sizeof(*' .. converted .. '));')
+
+        output:write('\n  } else {')
+        output:write(
+          '\n    api_set_error(error, kErrorTypeException, \
+          "Wrong type for argument '
+            .. j
+            .. ' when calling '
+            .. fn.name
+            .. ', expecting '
+            .. param[1]
+            .. '");'
+        )
+        output:write('\n    goto cleanup;')
+        output:write('\n  }\n')
+      else
+        if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then
+          -- Buffer, Window, and Tabpage have a specific type, but are stored in integer
+          output:write(
+            '\n  if (args.items['
+              .. (j - 1)
+              .. '].type == kObjectType'
+              .. rt
+              .. ' && args.items['
+              .. (j - 1)
+              .. '].data.integer >= 0) {'
+          )
+          output:write(
+            '\n    ' .. converted .. ' = (handle_T)args.items[' .. (j - 1) .. '].data.integer;'
+          )
+        else
+          output:write('\n  if (args.items[' .. (j - 1) .. '].type == kObjectType' .. rt .. ') {')
+          output:write(
+            '\n    '
+              .. converted
+              .. ' = args.items['
+              .. (j - 1)
+              .. '].data.'
+              .. attr_name(rt)
+              .. ';'
+          )
+        end
+        if
+          rt:match('^Buffer$')
+          or rt:match('^Window$')
+          or rt:match('^Tabpage$')
+          or rt:match('^Boolean$')
+        then
+          -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages
+          output:write(
+            '\n  } else if (args.items['
+              .. (j - 1)
+              .. '].type == kObjectTypeInteger && args.items['
+              .. (j - 1)
+              .. '].data.integer >= 0) {'
+          )
+          output:write(
+            '\n    ' .. converted .. ' = (handle_T)args.items[' .. (j - 1) .. '].data.integer;'
+          )
+        end
+        if rt:match('^Float$') then
+          -- accept integers for Floats
+          output:write('\n  } else if (args.items[' .. (j - 1) .. '].type == kObjectTypeInteger) {')
+          output:write(
+            '\n    ' .. converted .. ' = (Float)args.items[' .. (j - 1) .. '].data.integer;'
+          )
+        end
+        -- accept empty lua tables as empty dictionaries
+        if rt:match('^Dict') then
+          output:write(
+            '\n  } else if (args.items['
+              .. (j - 1)
+              .. '].type == kObjectTypeArray && args.items['
+              .. (j - 1)
+              .. '].data.array.size == 0) {'
+          ) --luacheck: ignore 631
+          output:write('\n    ' .. converted .. ' = (Dict)ARRAY_DICT_INIT;')
+        end
+        output:write('\n  } else {')
+        output:write(
+          '\n    api_set_error(error, kErrorTypeException, \
+          "Wrong type for argument '
+            .. j
+            .. ' when calling '
+            .. fn.name
+            .. ', expecting '
+            .. param[1]
+            .. '");'
+        )
+        output:write('\n    goto cleanup;')
+        output:write('\n  }\n')
+      end
+      args[#args + 1] = converted
+    end
+
+    if fn.textlock then
+      output:write('\n  if (text_locked()) {')
+      output:write('\n    api_set_error(error, kErrorTypeException, "%s", get_text_locked_msg());')
+      output:write('\n    goto cleanup;')
+      output:write('\n  }\n')
+    elseif fn.textlock_allow_cmdwin then
+      output:write('\n  if (textlock != 0 || expr_map_locked()) {')
+      output:write('\n    api_set_error(error, kErrorTypeException, "%s", e_textlock);')
+      output:write('\n    goto cleanup;')
+      output:write('\n  }\n')
+    end
+
+    -- function call
+    output:write('\n  ')
+    if fn.return_type ~= 'void' then
+      -- has a return value, prefix the call with a declaration
+      output:write(fn.return_type .. ' rv = ')
+    end
+
+    -- write the function name and the opening parenthesis
+    output:write(fn.name .. '(')
+
+    local call_args = {}
+    if fn.receives_channel_id then
+      table.insert(call_args, 'channel_id')
+    end
+
+    if fn.receives_array_args then
+      table.insert(call_args, 'args')
+    end
+
+    for _, a in ipairs(args) do
+      table.insert(call_args, a)
+    end
+
+    if fn.receives_arena then
+      table.insert(call_args, 'arena')
+    end
+
+    if fn.has_lua_imp then
+      table.insert(call_args, 'NULL')
+    end
+
+    if fn.can_fail then
+      table.insert(call_args, 'error')
+    end
+
+    output:write(table.concat(call_args, ', '))
+    output:write(');\n')
+
+    if fn.can_fail then
+      -- if the function can fail, also pass a pointer to the local error object
+      -- and check for the error
+      output:write('\n  if (ERROR_SET(error)) {')
+      output:write('\n    goto cleanup;')
+      output:write('\n  }\n')
+    end
+
+    local ret_type = real_type(fn.return_type)
+    if string.match(ret_type, '^KeyDict_') then
+      local table = string.sub(ret_type, 9) .. '_table'
+      output:write(
+        '\n  ret = DICT_OBJ(api_keydict_to_dict(&rv, '
+          .. table
+          .. ', ARRAY_SIZE('
+          .. table
+          .. '), arena));'
+      )
+    elseif ret_type ~= 'void' then
+      output:write('\n  ret = ' .. string.upper(real_type(fn.return_type)) .. '_OBJ(rv);')
+    end
+    output:write('\n\ncleanup:')
+
+    output:write('\n  return ret;\n}\n\n')
+  end
+end
+
+local remote_fns = {}
+for _, fn in ipairs(functions) do
+  if fn.remote then
+    remote_fns[fn.name] = fn
+  end
+end
+remote_fns.redraw = { impl_name = 'ui_client_redraw', fast = true }
+
+local names = vim.tbl_keys(remote_fns)
+table.sort(names)
+local hashorder, hashfun = hashy.hashy_hash('msgpack_rpc_get_handler_for', names, function(idx)
+  return 'method_handlers[' .. idx .. '].name'
+end)
+
+output:write('const MsgpackRpcRequestHandler method_handlers[] = {\n')
+for n, name in ipairs(hashorder) do
+  local fn = remote_fns[name]
+  fn.handler_id = n - 1
+  output:write(
+    '  { .name = "'
+      .. name
+      .. '", .fn = handle_'
+      .. (fn.impl_name or fn.name)
+      .. ', .fast = '
+      .. tostring(fn.fast)
+      .. ', .ret_alloc = '
+      .. tostring(not not fn.ret_alloc)
+      .. '},\n'
+  )
+end
+output:write('};\n\n')
+output:write(hashfun)
+
+output:close()
+
+functions.keysets = keysets
+local mpack_output = assert(io.open(mpack_outputf, 'wb'))
+mpack_output:write(mpack.encode(functions))
+mpack_output:close()
+
+local function include_headers(output_handle, headers_to_include)
+  for i = 1, #headers_to_include do
+    if headers_to_include[i]:sub(-12) ~= '.generated.h' then
+      output_handle:write('\n#include "nvim/' .. headers_to_include[i] .. '"')
+    end
+  end
+end
+
+local function write_shifted_output(str, ...)
+  str = str:gsub('\n  ', '\n')
+  str = str:gsub('^  ', '')
+  str = str:gsub(' +$', '')
+  output:write(string.format(str, ...))
+end
+
+-- start building lua output
+output = assert(io.open(lua_c_bindings_outputf, 'wb'))
+
+include_headers(output, headers)
+output:write('\n')
+
+local lua_c_functions = {}
+
+local function process_function(fn)
+  local lua_c_function_name = ('nlua_api_%s'):format(fn.name)
+  write_shifted_output(
+    [[
+
+  static int %s(lua_State *lstate)
+  {
+    Error err = ERROR_INIT;
+    Arena arena = ARENA_EMPTY;
+    char *err_param = 0;
+    if (lua_gettop(lstate) != %i) {
+      api_set_error(&err, kErrorTypeValidation, "Expected %i argument%s");
+      goto exit_0;
+    }
+  ]],
+    lua_c_function_name,
+    #fn.parameters,
+    #fn.parameters,
+    (#fn.parameters == 1) and '' or 's'
+  )
+  lua_c_functions[#lua_c_functions + 1] = {
+    binding = lua_c_function_name,
+    api = fn.name,
+  }
+
+  if not fn.fast then
+    write_shifted_output(
+      [[
+    if (!nlua_is_deferred_safe()) {
+      return luaL_error(lstate, e_fast_api_disabled, "%s");
+    }
+    ]],
+      fn.name
+    )
+  end
+
+  if fn.textlock then
+    write_shifted_output([[
+    if (text_locked()) {
+      api_set_error(&err, kErrorTypeException, "%%s", get_text_locked_msg());
+      goto exit_0;
+    }
+    ]])
+  elseif fn.textlock_allow_cmdwin then
+    write_shifted_output([[
+    if (textlock != 0 || expr_map_locked()) {
+      api_set_error(&err, kErrorTypeException, "%%s", e_textlock);
+      goto exit_0;
+    }
+    ]])
+  end
+
+  local cparams = ''
+  local free_code = {}
+  for j = #fn.parameters, 1, -1 do
+    local param = fn.parameters[j]
+    local cparam = string.format('arg%u', j)
+    local param_type = real_type(param[1])
+    local extra = param_type == 'Dict' and 'false, ' or ''
+    local arg_free_code = ''
+    if param[1] == 'Object' then
+      extra = 'true, '
+      arg_free_code = '  api_luarefs_free_object(' .. cparam .. ');'
+    elseif param[1] == 'DictOf(LuaRef)' then
+      extra = 'true, '
+      arg_free_code = '  api_luarefs_free_dict(' .. cparam .. ');'
+    elseif param[1] == 'LuaRef' then
+      arg_free_code = '  api_free_luaref(' .. cparam .. ');'
+    end
+    local errshift = 0
+    local seterr = ''
+    if string.match(param_type, '^KeyDict_') then
+      write_shifted_output(
+        [[
+    %s %s = KEYDICT_INIT;
+    nlua_pop_keydict(lstate, &%s, %s_get_field, &err_param, &arena, &err);
+    ]],
+        param_type,
+        cparam,
+        cparam,
+        param_type
+      )
+      cparam = '&' .. cparam
+      errshift = 1 -- free incomplete dict on error
+      arg_free_code = '  api_luarefs_free_keydict('
+        .. cparam
+        .. ', '
+        .. string.sub(param_type, 9)
+        .. '_table);'
+    else
+      write_shifted_output(
+        [[
+    const %s %s = nlua_pop_%s(lstate, %s&arena, &err);]],
+        param[1],
+        cparam,
+        param_type,
+        extra
+      )
+      seterr = '\n      err_param = "' .. param[2] .. '";'
+    end
+
+    write_shifted_output([[
+
+    if (ERROR_SET(&err)) {]] .. seterr .. [[
+
+      goto exit_%u;
+    }
+
+    ]], #fn.parameters - j + errshift)
+    free_code[#free_code + 1] = arg_free_code
+    cparams = cparam .. ', ' .. cparams
+  end
+  if fn.receives_channel_id then
+    cparams = 'LUA_INTERNAL_CALL, ' .. cparams
+  end
+  if fn.receives_arena then
+    cparams = cparams .. '&arena, '
+  end
+
+  if fn.has_lua_imp then
+    cparams = cparams .. 'lstate, '
+  end
+
+  if fn.can_fail then
+    cparams = cparams .. '&err'
+  else
+    cparams = cparams:gsub(', $', '')
+  end
+  local free_at_exit_code = ''
+  for i = 1, #free_code do
+    local rev_i = #free_code - i + 1
+    local code = free_code[rev_i]
+    if i == 1 and not string.match(real_type(fn.parameters[1][1]), '^KeyDict_') then
+      free_at_exit_code = free_at_exit_code .. ('\n%s'):format(code)
+    else
+      free_at_exit_code = free_at_exit_code .. ('\nexit_%u:\n%s'):format(rev_i, code)
+    end
+  end
+  local err_throw_code = [[
+
+exit_0:
+  arena_mem_free(arena_finish(&arena));
+  if (ERROR_SET(&err)) {
+    luaL_where(lstate, 1);
+    if (err_param) {
+      lua_pushstring(lstate, "Invalid '");
+      lua_pushstring(lstate, err_param);
+      lua_pushstring(lstate, "': ");
+    }
+    lua_pushstring(lstate, err.msg);
+    api_clear_error(&err);
+    lua_concat(lstate, err_param ? 5 : 2);
+    return lua_error(lstate);
+  }
+]]
+  local return_type
+  if fn.return_type ~= 'void' then
+    if fn.return_type:match('^ArrayOf') then
+      return_type = 'Array'
+    else
+      return_type = fn.return_type
+    end
+    local free_retval = ''
+    if fn.ret_alloc then
+      free_retval = '  api_free_' .. return_type:lower() .. '(ret);'
+    end
+    write_shifted_output('    %s ret = %s(%s);\n', fn.return_type, fn.name, cparams)
+
+    local ret_type = real_type(fn.return_type)
+    local ret_mode = (ret_type == 'Object') and '&' or ''
+    if fn.has_lua_imp then
+      -- only push onto the Lua stack if we haven't already
+      write_shifted_output(
+        [[
+    if (lua_gettop(lstate) == 0) {
+      nlua_push_%s(lstate, %sret, kNluaPushSpecial | kNluaPushFreeRefs);
+    }
+      ]],
+        return_type,
+        ret_mode
+      )
+    elseif string.match(ret_type, '^KeyDict_') then
+      write_shifted_output(
+        '    nlua_push_keydict(lstate, &ret, %s_table);\n',
+        string.sub(ret_type, 9)
+      )
+    else
+      local special = (fn.since ~= nil and fn.since < 11)
+      write_shifted_output(
+        '    nlua_push_%s(lstate, %sret, %s | kNluaPushFreeRefs);\n',
+        return_type,
+        ret_mode,
+        special and 'kNluaPushSpecial' or '0'
+      )
+    end
+
+    -- NOTE: we currently assume err_throw needs nothing from arena
+    write_shifted_output(
+      [[
+  %s
+  %s
+  %s
+    return 1;
+    ]],
+      free_retval,
+      free_at_exit_code,
+      err_throw_code
+    )
+  else
+    write_shifted_output(
+      [[
+    %s(%s);
+  %s
+  %s
+    return 0;
+    ]],
+      fn.name,
+      cparams,
+      free_at_exit_code,
+      err_throw_code
+    )
+  end
+  write_shifted_output([[
+  }
+  ]])
+end
+
+for _, fn in ipairs(functions) do
+  if fn.lua or fn.name:sub(1, 4) == '_vim' then
+    process_function(fn)
+  end
+end
+
+output:write(string.format(
+  [[
+void nlua_add_api_functions(lua_State *lstate)
+{
+  lua_createtable(lstate, 0, %u);
+]],
+  #lua_c_functions
+))
+for _, func in ipairs(lua_c_functions) do
+  output:write(string.format(
+    [[
+
+  lua_pushcfunction(lstate, &%s);
+  lua_setfield(lstate, -2, "%s");]],
+    func.binding,
+    func.api
+  ))
+end
+output:write([[
+
+  lua_setfield(lstate, -2, "api");
+}
+]])
+
+output:close()
+keysets_defs:close()

src/gen/gen_api_ui_events.lua

@@ -0,0 +1,219 @@
+local mpack = vim.mpack
+
+assert(#arg == 5)
+local input = io.open(arg[1], 'rb')
+local call_output = io.open(arg[2], 'wb')
+local remote_output = io.open(arg[3], 'wb')
+local metadata_output = io.open(arg[4], 'wb')
+local client_output = io.open(arg[5], 'wb')
+
+local c_grammar = require('gen.c_grammar')
+local events = c_grammar.grammar:match(input:read('*all'))
+
+local hashy = require 'gen.hashy'
+
+local function write_signature(output, ev, prefix, notype)
+  output:write('(' .. prefix)
+  if prefix == '' and #ev.parameters == 0 then
+    output:write('void')
+  end
+  for j = 1, #ev.parameters do
+    if j > 1 or prefix ~= '' then
+      output:write(', ')
+    end
+    local param = ev.parameters[j]
+    if not notype then
+      output:write(param[1] .. ' ')
+    end
+    output:write(param[2])
+  end
+  output:write(')')
+end
+
+local function write_arglist(output, ev)
+  if #ev.parameters == 0 then
+    return
+  end
+  output:write('  MAXSIZE_TEMP_ARRAY(args, ' .. #ev.parameters .. ');\n')
+  for j = 1, #ev.parameters do
+    local param = ev.parameters[j]
+    local kind = string.upper(param[1])
+    output:write('  ADD_C(args, ')
+    output:write(kind .. '_OBJ(' .. param[2] .. ')')
+    output:write(');\n')
+  end
+end
+
+local function call_ui_event_method(output, ev)
+  output:write('void ui_client_event_' .. ev.name .. '(Array args)\n{\n')
+
+  local hlattrs_args_count = 0
+  if #ev.parameters > 0 then
+    output:write('  if (args.size < ' .. #ev.parameters)
+    for j = 1, #ev.parameters do
+      local kind = ev.parameters[j][1]
+      if kind ~= 'Object' then
+        if kind == 'HlAttrs' then
+          kind = 'Dict'
+        end
+        output:write('\n      || args.items[' .. (j - 1) .. '].type != kObjectType' .. kind .. '')
+      end
+    end
+    output:write(') {\n')
+    output:write('    ELOG("Error handling ui event \'' .. ev.name .. '\'");\n')
+    output:write('    return;\n')
+    output:write('  }\n')
+  end
+
+  for j = 1, #ev.parameters do
+    local param = ev.parameters[j]
+    local kind = param[1]
+    output:write('  ' .. kind .. ' arg_' .. j .. ' = ')
+    if kind == 'HlAttrs' then
+      -- The first HlAttrs argument is rgb_attrs and second is cterm_attrs
+      output:write(
+        'ui_client_dict2hlattrs(args.items['
+          .. (j - 1)
+          .. '].data.dict, '
+          .. (hlattrs_args_count == 0 and 'true' or 'false')
+          .. ');\n'
+      )
+      hlattrs_args_count = hlattrs_args_count + 1
+    elseif kind == 'Object' then
+      output:write('args.items[' .. (j - 1) .. '];\n')
+    elseif kind == 'Window' then
+      output:write('(Window)args.items[' .. (j - 1) .. '].data.integer;\n')
+    else
+      output:write('args.items[' .. (j - 1) .. '].data.' .. string.lower(kind) .. ';\n')
+    end
+  end
+
+  output:write('  tui_' .. ev.name .. '(tui')
+  for j = 1, #ev.parameters do
+    output:write(', arg_' .. j)
+  end
+  output:write(');\n')
+
+  output:write('}\n\n')
+end
+
+events = vim.tbl_filter(function(ev)
+  return ev[1] ~= 'empty' and ev[1] ~= 'preproc'
+end, events)
+
+for i = 1, #events do
+  local ev = events[i]
+  assert(ev.return_type == 'void')
+
+  if ev.since == nil and not ev.noexport then
+    print('Ui event ' .. ev.name .. ' lacks since field.\n')
+    os.exit(1)
+  end
+  ev.since = tonumber(ev.since)
+
+  local args = #ev.parameters > 0 and 'args' or 'noargs'
+  if not ev.remote_only then
+    if not ev.remote_impl and not ev.noexport then
+      remote_output:write('void remote_ui_' .. ev.name)
+      write_signature(remote_output, ev, 'RemoteUI *ui')
+      remote_output:write('\n{\n')
+      write_arglist(remote_output, ev)
+      remote_output:write('  push_call(ui, "' .. ev.name .. '", ' .. args .. ');\n')
+      remote_output:write('}\n\n')
+    end
+  end
+
+  if not (ev.remote_only and ev.remote_impl) then
+    call_output:write('void ui_call_' .. ev.name)
+    write_signature(call_output, ev, '')
+    call_output:write('\n{\n')
+    if ev.remote_only then
+      -- Lua callbacks may emit other events or the same event again. Avoid the latter
+      -- by adding a recursion guard to each generated function that may call a Lua callback.
+      call_output:write('  static bool entered = false;\n')
+      call_output:write('  if (entered) {\n')
+      call_output:write('    return;\n')
+      call_output:write('  }\n')
+      call_output:write('  entered = true;\n')
+      write_arglist(call_output, ev)
+      call_output:write(('  ui_call_event("%s", %s, %s)'):format(ev.name, tostring(ev.fast), args))
+      call_output:write(';\n  entered = false;\n')
+    elseif ev.compositor_impl then
+      call_output:write('  ui_comp_' .. ev.name)
+      write_signature(call_output, ev, '', true)
+      call_output:write(';\n')
+      call_output:write('  UI_CALL')
+      write_signature(call_output, ev, '!ui->composed, ' .. ev.name .. ', ui', true)
+      call_output:write(';\n')
+    else
+      call_output:write('  UI_CALL')
+      write_signature(call_output, ev, 'true, ' .. ev.name .. ', ui', true)
+      call_output:write(';\n')
+    end
+    call_output:write('}\n\n')
+  end
+
+  if ev.compositor_impl then
+    call_output:write('void ui_composed_call_' .. ev.name)
+    write_signature(call_output, ev, '')
+    call_output:write('\n{\n')
+    call_output:write('  UI_CALL')
+    write_signature(call_output, ev, 'ui->composed, ' .. ev.name .. ', ui', true)
+    call_output:write(';\n')
+    call_output:write('}\n\n')
+  end
+
+  if (not ev.remote_only) and not ev.noexport and not ev.client_impl and not ev.client_ignore then
+    call_ui_event_method(client_output, ev)
+  end
+end
+
+local client_events = {}
+for _, ev in ipairs(events) do
+  if (not ev.noexport) and ((not ev.remote_only) or ev.client_impl) and not ev.client_ignore then
+    client_events[ev.name] = ev
+  end
+end
+
+local hashorder, hashfun = hashy.hashy_hash(
+  'ui_client_handler',
+  vim.tbl_keys(client_events),
+  function(idx)
+    return 'event_handlers[' .. idx .. '].name'
+  end
+)
+
+client_output:write('static const UIClientHandler event_handlers[] = {\n')
+
+for _, name in ipairs(hashorder) do
+  client_output:write('  { .name = "' .. name .. '", .fn = ui_client_event_' .. name .. '},\n')
+end
+
+client_output:write('\n};\n\n')
+client_output:write(hashfun)
+
+call_output:close()
+remote_output:close()
+client_output:close()
+
+-- don't expose internal attributes like "impl_name" in public metadata
+local exported_attributes = { 'name', 'parameters', 'since', 'deprecated_since' }
+local exported_events = {}
+for _, ev in ipairs(events) do
+  local ev_exported = {}
+  for _, attr in ipairs(exported_attributes) do
+    ev_exported[attr] = ev[attr]
+  end
+  for _, p in ipairs(ev_exported.parameters) do
+    if p[1] == 'HlAttrs' or p[1] == 'Dict' then
+      -- TODO(justinmk): for back-compat, but do clients actually look at this?
+      p[1] = 'Dictionary'
+    end
+  end
+  if not ev.noexport then
+    exported_events[#exported_events + 1] = ev_exported
+  end
+end
+
+metadata_output:write(mpack.encode(exported_events))
+metadata_output:close()

src/gen/gen_char_blob.lua

@@ -0,0 +1,96 @@
+if arg[1] == '--help' then
+  print('Usage:')
+  print('  ' .. arg[0] .. ' [-c] target source varname [source varname]...')
+  print('')
+  print('Generates C file with big uint8_t blob.')
+  print('Blob will be stored in a static const array named varname.')
+  os.exit()
+end
+
+-- Recognized options:
+--   -c   compile Lua bytecode
+local options = {}
+
+while true do
+  local opt = string.match(arg[1], '^-(%w)')
+  if not opt then
+    break
+  end
+
+  options[opt] = true
+  table.remove(arg, 1)
+end
+
+assert(#arg >= 3 and (#arg - 1) % 2 == 0)
+
+local target_file = arg[1] or error('Need a target file')
+local target = io.open(target_file, 'w')
+
+target:write('#include <stdint.h>\n\n')
+
+local index_items = {}
+
+local warn_on_missing_compiler = true
+local modnames = {}
+for argi = 2, #arg, 2 do
+  local source_file = arg[argi]
+  local modname = arg[argi + 1]
+  if modnames[modname] then
+    error(string.format('modname %q is already specified for file %q', modname, modnames[modname]))
+  end
+  modnames[modname] = source_file
+
+  local varname = string.gsub(modname, '%.', '_dot_') .. '_module'
+  target:write(('static const uint8_t %s[] = {\n'):format(varname))
+
+  local output
+  if options.c then
+    local luac = os.getenv('LUAC_PRG')
+    if luac and luac ~= '' then
+      output = io.popen(luac:format(source_file), 'r'):read('*a')
+    elseif warn_on_missing_compiler then
+      print('LUAC_PRG is missing, embedding raw source')
+      warn_on_missing_compiler = false
+    end
+  end
+
+  if not output then
+    local source = io.open(source_file, 'r')
+      or error(string.format("source_file %q doesn't exist", source_file))
+    output = source:read('*a')
+    source:close()
+  end
+
+  local num_bytes = 0
+  local MAX_NUM_BYTES = 15 -- 78 / 5: maximum number of bytes on one line
+  target:write(' ')
+
+  local increase_num_bytes
+  increase_num_bytes = function()
+    num_bytes = num_bytes + 1
+    if num_bytes == MAX_NUM_BYTES then
+      num_bytes = 0
+      target:write('\n ')
+    end
+  end
+
+  for i = 1, string.len(output) do
+    local byte = output:byte(i)
+    target:write(string.format(' %3u,', byte))
+    increase_num_bytes()
+  end
+
+  target:write('  0};\n')
+  if modname ~= '_' then
+    table.insert(
+      index_items,
+      '  { "' .. modname .. '", ' .. varname .. ', sizeof ' .. varname .. ' },\n\n'
+    )
+  end
+end
+
+target:write('static ModuleDef builtin_modules[] = {\n')
+target:write(table.concat(index_items))
+target:write('};\n')
+
+target:close()

src/gen/gen_declarations.lua

@@ -0,0 +1,186 @@
+local grammar = require('gen.c_grammar').grammar
+
+--- @param fname string
+--- @return string?
+local function read_file(fname)
+  local f = io.open(fname, 'r')
+  if not f then
+    return
+  end
+  local contents = f:read('*a')
+  f:close()
+  return contents
+end
+
+--- @param fname string
+--- @param contents string[]
+local function write_file(fname, contents)
+  local contents_s = table.concat(contents, '\n') .. '\n'
+  local fcontents = read_file(fname)
+  if fcontents == contents_s then
+    return
+  end
+  local f = assert(io.open(fname, 'w'))
+  f:write(contents_s)
+  f:close()
+end
+
+--- @param fname string
+--- @param non_static_fname string
+--- @return string? non_static
+local function add_iwyu_non_static(fname, non_static_fname)
+  if fname:find('.*/src/nvim/.*%.c$') then
+    -- Add an IWYU pragma comment if the corresponding .h file exists.
+    local header_fname = fname:sub(1, -3) .. '.h'
+    local header_f = io.open(header_fname, 'r')
+    if header_f then
+      header_f:close()
+      return (header_fname:gsub('.*/src/nvim/', 'nvim/'))
+    end
+  elseif non_static_fname:find('/include/api/private/dispatch_wrappers%.h%.generated%.h$') then
+    return 'nvim/api/private/dispatch.h'
+  elseif non_static_fname:find('/include/ui_events_call%.h%.generated%.h$') then
+    return 'nvim/ui.h'
+  elseif non_static_fname:find('/include/ui_events_client%.h%.generated%.h$') then
+    return 'nvim/ui_client.h'
+  elseif non_static_fname:find('/include/ui_events_remote%.h%.generated%.h$') then
+    return 'nvim/api/ui.h'
+  end
+end
+
+--- @param d string
+local function process_decl(d)
+  -- Comments are really handled by preprocessor, so the following is not
+  -- needed
+  d = d:gsub('/%*.-%*/', '')
+  d = d:gsub('//.-\n', '\n')
+  d = d:gsub('# .-\n', '')
+  d = d:gsub('\n', ' ')
+  d = d:gsub('%s+', ' ')
+  d = d:gsub(' ?%( ?', '(')
+  d = d:gsub(' ?, ?', ', ')
+  d = d:gsub(' ?(%*+) ?', ' %1')
+  d = d:gsub(' ?(FUNC_ATTR_)', ' %1')
+  d = d:gsub(' $', '')
+  d = d:gsub('^ ', '')
+  return d .. ';'
+end
+
+--- @param fname string
+--- @param text string
+--- @return string[] static
+--- @return string[] non_static
+--- @return boolean any_static
+local function gen_declarations(fname, text)
+  local non_static = {} --- @type string[]
+  local static = {} --- @type string[]
+
+  local neededfile = fname:match('[^/]+$')
+  local curfile = nil
+  local any_static = false
+  for _, node in ipairs(grammar:match(text)) do
+    if node[1] == 'preproc' then
+      curfile = node.content:match('^%a* %d+ "[^"]-/?([^"/]+)"') or curfile
+    elseif node[1] == 'proto' and curfile == neededfile then
+      local node_text = text:sub(node.pos, node.endpos - 1)
+      local declaration = process_decl(node_text)
+
+      if node.static then
+        if not any_static and declaration:find('FUNC_ATTR_') then
+          any_static = true
+        end
+        static[#static + 1] = declaration
+      else
+        non_static[#non_static + 1] = 'DLLEXPORT ' .. declaration
+      end
+    end
+  end
+
+  return static, non_static, any_static
+end
+
+local usage = [[
+Usage:
+
+    gen_declarations.lua definitions.c static.h non-static.h definitions.i
+
+Generates declarations for a C file definitions.c, putting declarations for
+static functions into static.h and declarations for non-static functions into
+non-static.h. File `definitions.i' should contain an already preprocessed
+version of definitions.c and it is the only one which is actually parsed,
+definitions.c is needed only to determine functions from which file out of all
+functions found in definitions.i are needed and to generate an IWYU comment.
+]]
+
+local function main()
+  local fname = arg[1]
+  local static_fname = arg[2]
+  local non_static_fname = arg[3]
+  local preproc_fname = arg[4]
+  local static_basename = arg[5]
+
+  if fname == '--help' or #arg < 5 then
+    print(usage)
+    os.exit()
+  end
+
+  local text = assert(read_file(preproc_fname))
+
+  local static_decls, non_static_decls, any_static = gen_declarations(fname, text)
+
+  local static = {} --- @type string[]
+  if fname:find('.*/src/nvim/.*%.h$') then
+    static[#static + 1] = ('// IWYU pragma: private, include "%s"'):format(
+      fname:gsub('.*/src/nvim/', 'nvim/')
+    )
+  end
+  vim.list_extend(static, {
+    '#define DEFINE_FUNC_ATTRIBUTES',
+    '#include "nvim/func_attr.h"',
+    '#undef DEFINE_FUNC_ATTRIBUTES',
+  })
+  vim.list_extend(static, static_decls)
+  vim.list_extend(static, {
+    '#define DEFINE_EMPTY_ATTRIBUTES',
+    '#include "nvim/func_attr.h"  // IWYU pragma: export',
+    '',
+  })
+
+  write_file(static_fname, static)
+
+  if any_static then
+    local orig_text = assert(read_file(fname))
+    local pat = '\n#%s?include%s+"' .. static_basename .. '"\n'
+    local pat_comment = '\n#%s?include%s+"' .. static_basename .. '"%s*//'
+    if not orig_text:find(pat) and not orig_text:find(pat_comment) then
+      error(('fail: missing include for %s in %s'):format(static_basename, fname))
+    end
+  end
+
+  if non_static_fname ~= 'SKIP' then
+    local non_static = {} --- @type string[]
+    local iwyu_non_static = add_iwyu_non_static(fname, non_static_fname)
+    if iwyu_non_static then
+      non_static[#non_static + 1] = ('// IWYU pragma: private, include "%s"'):format(
+        iwyu_non_static
+      )
+    end
+    vim.list_extend(non_static, {
+      '#define DEFINE_FUNC_ATTRIBUTES',
+      '#include "nvim/func_attr.h"',
+      '#undef DEFINE_FUNC_ATTRIBUTES',
+      '#ifndef DLLEXPORT',
+      '#  ifdef MSWIN',
+      '#    define DLLEXPORT __declspec(dllexport)',
+      '#  else',
+      '#    define DLLEXPORT',
+      '#  endif',
+      '#endif',
+    })
+    vim.list_extend(non_static, non_static_decls)
+    non_static[#non_static + 1] = '#include "nvim/func_attr.h"'
+    write_file(non_static_fname, non_static)
+  end
+end
+
+return main()

src/gen/gen_eval.lua

@@ -0,0 +1,112 @@
+local mpack = vim.mpack
+
+local autodir = arg[1]
+local metadata_file = arg[2]
+local funcs_file = arg[3]
+
+local funcsfname = autodir .. '/funcs.generated.h'
+
+--Will generate funcs.generated.h with definition of functions static const array.
+
+local hashy = require 'gen.hashy'
+
+local hashpipe = assert(io.open(funcsfname, 'wb'))
+
+hashpipe:write([[
+#include "nvim/arglist.h"
+#include "nvim/cmdexpand.h"
+#include "nvim/cmdhist.h"
+#include "nvim/digraph.h"
+#include "nvim/eval.h"
+#include "nvim/eval/buffer.h"
+#include "nvim/eval/deprecated.h"
+#include "nvim/eval/fs.h"
+#include "nvim/eval/funcs.h"
+#include "nvim/eval/typval.h"
+#include "nvim/eval/vars.h"
+#include "nvim/eval/window.h"
+#include "nvim/ex_docmd.h"
+#include "nvim/ex_getln.h"
+#include "nvim/fold.h"
+#include "nvim/getchar.h"
+#include "nvim/insexpand.h"
+#include "nvim/mapping.h"
+#include "nvim/match.h"
+#include "nvim/mbyte.h"
+#include "nvim/menu.h"
+#include "nvim/mouse.h"
+#include "nvim/move.h"
+#include "nvim/quickfix.h"
+#include "nvim/runtime.h"
+#include "nvim/search.h"
+#include "nvim/state.h"
+#include "nvim/strings.h"
+#include "nvim/sign.h"
+#include "nvim/testing.h"
+#include "nvim/undo.h"
+
+]])
+
+local funcs = require('nvim.eval').funcs
+for _, func in pairs(funcs) do
+  if func.float_func then
+    func.func = 'float_op_wrapper'
+    func.data = '{ .float_func = &' .. func.float_func .. ' }'
+  end
+end
+
+local metadata = mpack.decode(io.open(metadata_file, 'rb'):read('*all'))
+for _, fun in ipairs(metadata) do
+  if fun.eval then
+    funcs[fun.name] = {
+      args = #fun.parameters,
+      func = 'api_wrapper',
+      data = '{ .api_handler = &method_handlers[' .. fun.handler_id .. '] }',
+    }
+  end
+end
+
+local func_names = vim.tbl_filter(function(name)
+  return name:match('__%d*$') == nil
+end, vim.tbl_keys(funcs))
+
+table.sort(func_names)
+
+local funcsdata = assert(io.open(funcs_file, 'w'))
+funcsdata:write(mpack.encode(func_names))
+funcsdata:close()
+
+local neworder, hashfun = hashy.hashy_hash('find_internal_func', func_names, function(idx)
+  return 'functions[' .. idx .. '].name'
+end)
+
+hashpipe:write('static const EvalFuncDef functions[] = {\n')
+
+for _, name in ipairs(neworder) do
+  local def = funcs[name]
+  local args = def.args or 0
+  if type(args) == 'number' then
+    args = { args, args }
+  elseif #args == 1 then
+    args[2] = 'MAX_FUNC_ARGS'
+  end
+  local base = def.base or 'BASE_NONE'
+  local func = def.func or ('f_' .. name)
+  local data = def.data or '{ .null = NULL }'
+  local fast = def.fast and 'true' or 'false'
+  hashpipe:write(
+    ('  { "%s", %s, %s, %s, %s, &%s, %s },\n'):format(
+      name,
+      args[1],
+      args[2],
+      base,
+      fast,
+      func,
+      data
+    )
+  )
+end
+hashpipe:write('  { NULL, 0, 0, BASE_NONE, false, NULL, { .null = NULL } },\n')
+hashpipe:write('};\n\n')
+hashpipe:write(hashfun)
+hashpipe:close()

src/gen/gen_events.lua

@@ -0,0 +1,42 @@
+local fileio_enum_file = arg[1]
+local names_file = arg[2]
+
+local auevents = require('nvim.auevents')
+local events = auevents.events
+
+local enum_tgt = io.open(fileio_enum_file, 'w')
+local names_tgt = io.open(names_file, 'w')
+
+enum_tgt:write([[
+// IWYU pragma: private, include "nvim/autocmd_defs.h"
+
+typedef enum auto_event {]])
+names_tgt:write([[
+static const struct event_name {
+  size_t len;
+  char *name;
+  int event;
+} event_names[] = {]])
+
+local aliases = 0
+for i, event in ipairs(events) do
+  enum_tgt:write(('\n  EVENT_%s = %u,'):format(event[1]:upper(), i + aliases - 1))
+  -- Events with positive keys aren't allowed in 'eventignorewin'.
+  local event_int = ('%sEVENT_%s'):format(event[3] and '-' or '', event[1]:upper())
+  names_tgt:write(('\n  {%u, "%s", %s},'):format(#event[1], event[1], event_int))
+  for _, alias in ipairs(event[2]) do
+    aliases = aliases + 1
+    names_tgt:write(('\n  {%u, "%s", %s},'):format(#alias, alias, event_int))
+    enum_tgt:write(('\n  EVENT_%s = %u,'):format(alias:upper(), i + aliases - 1))
+  end
+  if i == #events then -- Last item.
+    enum_tgt:write(('\n  NUM_EVENTS = %u,'):format(i + aliases))
+  end
+end
+
+names_tgt:write('\n  {0, NULL, (event_T)0},\n};\n')
+names_tgt:write('\nstatic AutoCmdVec autocmds[NUM_EVENTS] = { 0 };\n')
+names_tgt:close()
+
+enum_tgt:write('\n} event_T;\n')
+enum_tgt:close()

src/gen/gen_ex_cmds.lua

@@ -0,0 +1,194 @@
+local includedir = arg[1]
+local autodir = arg[2]
+
+-- Will generate files ex_cmds_enum.generated.h with cmdidx_T enum
+-- and ex_cmds_defs.generated.h with main Ex commands definitions.
+
+local enumfname = includedir .. '/ex_cmds_enum.generated.h'
+local defsfname = autodir .. '/ex_cmds_defs.generated.h'
+
+local enumfile = io.open(enumfname, 'w')
+local defsfile = io.open(defsfname, 'w')
+
+local bit = require 'bit'
+local ex_cmds = require('nvim.ex_cmds')
+local defs = ex_cmds.cmds
+local flags = ex_cmds.flags
+
+local byte_a = string.byte('a')
+local byte_z = string.byte('z')
+local a_to_z = byte_z - byte_a + 1
+
+-- Table giving the index of the first command in cmdnames[] to lookup
+-- based on the first letter of a command.
+local cmdidxs1_out = string.format(
+  [[
+static const uint16_t cmdidxs1[%u] = {
+]],
+  a_to_z
+)
+-- Table giving the index of the first command in cmdnames[] to lookup
+-- based on the first 2 letters of a command.
+-- Values in cmdidxs2[c1][c2] are relative to cmdidxs1[c1] so that they
+-- fit in a byte.
+local cmdidxs2_out = string.format(
+  [[
+static const uint8_t cmdidxs2[%u][%u] = {
+  /*           a   b   c   d   e   f   g   h   i   j   k   l   m   n   o   p   q   r   s   t   u   v   w   x   y   z */
+]],
+  a_to_z,
+  a_to_z
+)
+
+enumfile:write([[
+// IWYU pragma: private, include "nvim/ex_cmds_defs.h"
+
+typedef enum CMD_index {
+]])
+defsfile:write(string.format(
+  [[
+#include "nvim/arglist.h"
+#include "nvim/autocmd.h"
+#include "nvim/buffer.h"
+#include "nvim/cmdhist.h"
+#include "nvim/debugger.h"
+#include "nvim/diff.h"
+#include "nvim/digraph.h"
+#include "nvim/eval.h"
+#include "nvim/eval/userfunc.h"
+#include "nvim/eval/vars.h"
+#include "nvim/ex_cmds.h"
+#include "nvim/ex_cmds2.h"
+#include "nvim/ex_docmd.h"
+#include "nvim/ex_eval.h"
+#include "nvim/ex_session.h"
+#include "nvim/help.h"
+#include "nvim/indent.h"
+#include "nvim/lua/executor.h"
+#include "nvim/lua/secure.h"
+#include "nvim/mapping.h"
+#include "nvim/mark.h"
+#include "nvim/match.h"
+#include "nvim/menu.h"
+#include "nvim/message.h"
+#include "nvim/ops.h"
+#include "nvim/option.h"
+#include "nvim/os/lang.h"
+#include "nvim/profile.h"
+#include "nvim/quickfix.h"
+#include "nvim/runtime.h"
+#include "nvim/sign.h"
+#include "nvim/spell.h"
+#include "nvim/spellfile.h"
+#include "nvim/syntax.h"
+#include "nvim/undo.h"
+#include "nvim/usercmd.h"
+#include "nvim/version.h"
+
+static const int command_count = %u;
+static CommandDefinition cmdnames[%u] = {
+]],
+  #defs,
+  #defs
+))
+local cmds, cmdidxs1, cmdidxs2 = {}, {}, {}
+for _, cmd in ipairs(defs) do
+  if bit.band(cmd.flags, flags.RANGE) == flags.RANGE then
+    assert(
+      cmd.addr_type ~= 'ADDR_NONE',
+      string.format('ex_cmds.lua:%s: Using RANGE with ADDR_NONE\n', cmd.command)
+    )
+  else
+    assert(
+      cmd.addr_type == 'ADDR_NONE',
+      string.format('ex_cmds.lua:%s: Missing ADDR_NONE\n', cmd.command)
+    )
+  end
+  if bit.band(cmd.flags, flags.DFLALL) == flags.DFLALL then
+    assert(
+      cmd.addr_type ~= 'ADDR_OTHER' and cmd.addr_type ~= 'ADDR_NONE',
+      string.format('ex_cmds.lua:%s: Missing misplaced DFLALL\n', cmd.command)
+    )
+  end
+  if bit.band(cmd.flags, flags.PREVIEW) == flags.PREVIEW then
+    assert(
+      cmd.preview_func ~= nil,
+      string.format('ex_cmds.lua:%s: Missing preview_func\n', cmd.command)
+    )
+  end
+  local enumname = cmd.enum or ('CMD_' .. cmd.command)
+  local byte_cmd = cmd.command:sub(1, 1):byte()
+  if byte_a <= byte_cmd and byte_cmd <= byte_z then
+    table.insert(cmds, cmd.command)
+  end
+  local preview_func
+  if cmd.preview_func then
+    preview_func = string.format('&%s', cmd.preview_func)
+  else
+    preview_func = 'NULL'
+  end
+  enumfile:write('  ' .. enumname .. ',\n')
+  defsfile:write(string.format(
+    [[
+  [%s] = {
+    .cmd_name = "%s",
+    .cmd_func = (ex_func_T)&%s,
+    .cmd_preview_func = %s,
+    .cmd_argt = %uL,
+    .cmd_addr_type = %s
+  },
+]],
+    enumname,
+    cmd.command,
+    cmd.func,
+    preview_func,
+    cmd.flags,
+    cmd.addr_type
+  ))
+end
+for i = #cmds, 1, -1 do
+  local cmd = cmds[i]
+  -- First and second characters of the command
+  local c1 = cmd:sub(1, 1)
+  cmdidxs1[c1] = i - 1
+  if cmd:len() >= 2 then
+    local c2 = cmd:sub(2, 2)
+    local byte_c2 = string.byte(c2)
+    if byte_a <= byte_c2 and byte_c2 <= byte_z then
+      if not cmdidxs2[c1] then
+        cmdidxs2[c1] = {}
+      end
+      cmdidxs2[c1][c2] = i - 1
+    end
+  end
+end
+for i = byte_a, byte_z do
+  local c1 = string.char(i)
+  cmdidxs1_out = cmdidxs1_out .. '  /*  ' .. c1 .. '  */ ' .. cmdidxs1[c1] .. ',\n'
+  cmdidxs2_out = cmdidxs2_out .. '  /*  ' .. c1 .. '  */ {'
+  for j = byte_a, byte_z do
+    local c2 = string.char(j)
+    cmdidxs2_out = cmdidxs2_out
+      .. ((cmdidxs2[c1] and cmdidxs2[c1][c2]) and string.format(
+        '%3d',
+        cmdidxs2[c1][c2] - cmdidxs1[c1]
+      ) or '  0')
+      .. ','
+  end
+  cmdidxs2_out = cmdidxs2_out .. ' },\n'
+end
+enumfile:write([[
+  CMD_SIZE,
+  CMD_USER = -1,
+  CMD_USER_BUF = -2
+} cmdidx_T;
+]])
+defsfile:write(string.format(
+  [[
+};
+%s};
+%s};
+]],
+  cmdidxs1_out,
+  cmdidxs2_out
+))

src/gen/gen_filetype.lua

@@ -0,0 +1,209 @@
+local do_not_run = true
+if do_not_run then
+  print([[
+    This script was used to bootstrap the filetype patterns in runtime/lua/vim/filetype.lua. It
+    should no longer be used except for testing purposes. New filetypes, or changes to existing
+    filetypes, should be ported manually as part of the vim-patch process.
+  ]])
+  return
+end
+
+local filetype_vim = 'runtime/filetype.vim'
+local filetype_lua = 'runtime/lua/vim/filetype.lua'
+
+local keywords = {
+  ['for'] = true,
+  ['or'] = true,
+  ['and'] = true,
+  ['end'] = true,
+  ['do'] = true,
+  ['if'] = true,
+  ['while'] = true,
+  ['repeat'] = true,
+}
+
+local sections = {
+  extension = { str = {}, func = {} },
+  filename = { str = {}, func = {} },
+  pattern = { str = {}, func = {} },
+}
+
+local specialchars = '%*%?\\%$%[%]%{%}'
+
+local function add_pattern(pat, ft)
+  local ok = true
+
+  -- Patterns that start or end with { or } confuse splitting on commas and make parsing harder, so just skip those
+  if not string.find(pat, '^%{') and not string.find(pat, '%}$') then
+    for part in string.gmatch(pat, '[^,]+') do
+      if not string.find(part, '[' .. specialchars .. ']') then
+        if type(ft) == 'string' then
+          sections.filename.str[part] = ft
+        else
+          sections.filename.func[part] = ft
+        end
+      elseif string.match(part, '^%*%.[^%./' .. specialchars .. ']+$') then
+        if type(ft) == 'string' then
+          sections.extension.str[part:sub(3)] = ft
+        else
+          sections.extension.func[part:sub(3)] = ft
+        end
+      else
+        if string.match(part, '^%*/[^' .. specialchars .. ']+$') then
+          -- For patterns matching */some/pattern we want to easily match files
+          -- with path /some/pattern, so include those in filename detection
+          if type(ft) == 'string' then
+            sections.filename.str[part:sub(2)] = ft
+          else
+            sections.filename.func[part:sub(2)] = ft
+          end
+        end
+
+        if string.find(part, '^[%w-_.*?%[%]/]+$') then
+          local p = part:gsub('%.', '%%.'):gsub('%*', '.*'):gsub('%?', '.')
+          -- Insert into array to maintain order rather than setting
+          -- key-value directly
+          if type(ft) == 'string' then
+            sections.pattern.str[p] = ft
+          else
+            sections.pattern.func[p] = ft
+          end
+        else
+          ok = false
+        end
+      end
+    end
+  end
+
+  return ok
+end
+
+local function parse_line(line)
+  local pat, ft
+  pat, ft = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)')
+  if pat then
+    return add_pattern(pat, ft)
+  else
+    local func
+    pat, func = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)')
+    if pat then
+      return add_pattern(pat, function()
+        return func
+      end)
+    end
+  end
+end
+
+local unparsed = {}
+local full_line
+for line in io.lines(filetype_vim) do
+  local cont = string.match(line, '^%s*\\%s*(.*)$')
+  if cont then
+    full_line = full_line .. ' ' .. cont
+  else
+    if full_line then
+      if not parse_line(full_line) and string.find(full_line, '^%s*au%a* Buf') then
+        table.insert(unparsed, full_line)
+      end
+    end
+    full_line = line
+  end
+end
+
+if #unparsed > 0 then
+  print('Failed to parse the following patterns:')
+  for _, v in ipairs(unparsed) do
+    print(v)
+  end
+end
+
+local function add_item(indent, key, ft)
+  if type(ft) == 'string' then
+    if string.find(key, '%A') or keywords[key] then
+      key = string.format('["%s"]', key)
+    end
+    return string.format([[%s%s = "%s",]], indent, key, ft)
+  elseif type(ft) == 'function' then
+    local func = ft()
+    if string.find(key, '%A') or keywords[key] then
+      key = string.format('["%s"]', key)
+    end
+    -- Right now only a single argument is supported, which covers
+    -- everything in filetype.vim as of this writing
+    local arg = string.match(func, '%((.*)%)$')
+    func = string.gsub(func, '%(.*$', '')
+    if arg == '' then
+      -- Function with no arguments, call the function directly
+      return string.format([[%s%s = function() vim.fn["%s"]() end,]], indent, key, func)
+    elseif string.match(arg, [[^(["']).*%1$]]) then
+      -- String argument
+      if func == 's:StarSetf' then
+        return string.format([[%s%s = starsetf(%s),]], indent, key, arg)
+      else
+        return string.format([[%s%s = function() vim.fn["%s"](%s) end,]], indent, key, func, arg)
+      end
+    elseif string.find(arg, '%(') then
+      -- Function argument
+      return string.format(
+        [[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]],
+        indent,
+        key,
+        func,
+        arg
+      )
+    else
+      assert(false, arg)
+    end
+  end
+end
+
+do
+  local lines = {}
+  local start = false
+  for line in io.lines(filetype_lua) do
+    if line:match('^%s+-- END [A-Z]+$') then
+      start = false
+    end
+
+    if not start then
+      table.insert(lines, line)
+    end
+
+    local indent, section = line:match('^(%s+)-- BEGIN ([A-Z]+)$')
+    if section then
+      start = true
+      local t = sections[string.lower(section)]
+
+      local sorted = {}
+      for k, v in pairs(t.str) do
+        table.insert(sorted, { [k] = v })
+      end
+
+      table.sort(sorted, function(a, b)
+        return a[next(a)] < b[next(b)]
+      end)
+
+      for _, v in ipairs(sorted) do
+        local k = next(v)
+        table.insert(lines, add_item(indent, k, v[k]))
+      end
+
+      sorted = {}
+      for k, v in pairs(t.func) do
+        table.insert(sorted, { [k] = v })
+      end
+
+      table.sort(sorted, function(a, b)
+        return next(a) < next(b)
+      end)
+
+      for _, v in ipairs(sorted) do
+        local k = next(v)
+        table.insert(lines, add_item(indent, k, v[k]))
+      end
+    end
+  end
+  local f = io.open(filetype_lua, 'w')
+  f:write(table.concat(lines, '\n') .. '\n')
+  f:close()
+end

src/gen/gen_lsp.lua

@@ -0,0 +1,514 @@
+-- Generates lua-ls annotations for lsp.
+
+local USAGE = [[
+Generates lua-ls annotations for lsp.
+
+USAGE:
+nvim -l src/gen/gen_lsp.lua gen  # by default, this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua
+nvim -l src/gen/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua
+nvim -l src/gen/gen_lsp.lua gen --version 3.18 --methods --capabilities
+]]
+
+local DEFAULT_LSP_VERSION = '3.18'
+
+local M = {}
+
+local function tofile(fname, text)
+  local f = io.open(fname, 'w')
+  if not f then
+    error(('failed to write: %s'):format(f))
+  else
+    print(('Written to: %s'):format(fname))
+    f:write(text)
+    f:close()
+  end
+end
+
+--- The LSP protocol JSON data (it's partial, non-exhaustive).
+--- https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/3.18/metaModel/metaModel.schema.json
+--- @class vim._gen_lsp.Protocol
+--- @field requests vim._gen_lsp.Request[]
+--- @field notifications vim._gen_lsp.Notification[]
+--- @field structures vim._gen_lsp.Structure[]
+--- @field enumerations vim._gen_lsp.Enumeration[]
+--- @field typeAliases vim._gen_lsp.TypeAlias[]
+
+---@param opt vim._gen_lsp.opt
+---@return vim._gen_lsp.Protocol
+local function read_json(opt)
+  local uri = 'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/'
+    .. opt.version
+    .. '/metaModel/metaModel.json'
+  print('Reading ' .. uri)
+
+  local res = vim.system({ 'curl', '--no-progress-meter', uri, '-o', '-' }):wait()
+  if res.code ~= 0 or (res.stdout or ''):len() < 999 then
+    print(('URL failed: %s'):format(uri))
+    vim.print(res)
+    error(res.stdout)
+  end
+  return vim.json.decode(res.stdout)
+end
+
+-- Gets the Lua symbol for a given fully-qualified LSP method name.
+local function to_luaname(s)
+  -- "$/" prefix is special: https://microsoft.github.io/language-server-protocol/specification/#dollarRequests
+  return s:gsub('^%$', 'dollar'):gsub('/', '_')
+end
+
+---@param protocol vim._gen_lsp.Protocol
+---@param gen_methods boolean
+---@param gen_capabilities boolean
+local function write_to_protocol(protocol, gen_methods, gen_capabilities)
+  if not gen_methods and not gen_capabilities then
+    return
+  end
+
+  local indent = (' '):rep(2)
+
+  --- @class vim._gen_lsp.Request
+  --- @field deprecated? string
+  --- @field documentation? string
+  --- @field messageDirection string
+  --- @field clientCapability? string
+  --- @field serverCapability? string
+  --- @field method string
+  --- @field params? any
+  --- @field proposed? boolean
+  --- @field registrationMethod? string
+  --- @field registrationOptions? any
+  --- @field since? string
+
+  --- @class vim._gen_lsp.Notification
+  --- @field deprecated? string
+  --- @field documentation? string
+  --- @field errorData? any
+  --- @field messageDirection string
+  --- @field clientCapability? string
+  --- @field serverCapability? string
+  --- @field method string
+  --- @field params? any[]
+  --- @field partialResult? any
+  --- @field proposed? boolean
+  --- @field registrationMethod? string
+  --- @field registrationOptions? any
+  --- @field result any
+  --- @field since? string
+
+  ---@type (vim._gen_lsp.Request|vim._gen_lsp.Notification)[]
+  local all = vim.list_extend(protocol.requests, protocol.notifications)
+  table.sort(all, function(a, b)
+    return to_luaname(a.method) < to_luaname(b.method)
+  end)
+
+  local output = { '-- Generated by gen_lsp.lua, keep at end of file.' }
+
+  if gen_methods then
+    output[#output + 1] = '--- @alias vim.lsp.protocol.Method.ClientToServer'
+
+    for _, item in ipairs(all) do
+      if item.method and item.messageDirection == 'clientToServer' then
+        output[#output + 1] = ("--- | '%s',"):format(item.method)
+      end
+    end
+
+    vim.list_extend(output, {
+      '',
+      '--- @alias vim.lsp.protocol.Method.ServerToClient',
+    })
+    for _, item in ipairs(all) do
+      if item.method and item.messageDirection == 'serverToClient' then
+        output[#output + 1] = ("--- | '%s',"):format(item.method)
+      end
+    end
+
+    vim.list_extend(output, {
+      '',
+      '--- @alias vim.lsp.protocol.Method',
+      '--- | vim.lsp.protocol.Method.ClientToServer',
+      '--- | vim.lsp.protocol.Method.ServerToClient',
+      '',
+      '-- Generated by gen_lsp.lua, keep at end of file.',
+      '--- @enum vim.lsp.protocol.Methods',
+      '--- @see https://microsoft.github.io/language-server-protocol/specification/#metaModel',
+      '--- LSP method names.',
+      'protocol.Methods = {',
+    })
+
+    for _, item in ipairs(all) do
+      if item.method then
+        if item.documentation then
+          local document = vim.split(item.documentation, '\n?\n', { trimempty = true })
+          for _, docstring in ipairs(document) do
+            output[#output + 1] = indent .. '--- ' .. docstring
+          end
+        end
+        output[#output + 1] = ("%s%s = '%s',"):format(indent, to_luaname(item.method), item.method)
+      end
+    end
+    output[#output + 1] = '}'
+  end
+
+  if gen_capabilities then
+    vim.list_extend(output, {
+      '',
+      '-- stylua: ignore start',
+      '-- Generated by gen_lsp.lua, keep at end of file.',
+      '--- Maps method names to the required server capability',
+      'protocol._request_name_to_capability = {',
+    })
+
+    for _, item in ipairs(all) do
+      if item.serverCapability then
+        output[#output + 1] = ("%s['%s'] = { %s },"):format(
+          indent,
+          item.method,
+          table.concat(
+            vim
+              .iter(vim.split(item.serverCapability, '.', { plain = true }))
+              :map(function(segment)
+                return "'" .. segment .. "'"
+              end)
+              :totable(),
+            ', '
+          )
+        )
+      end
+    end
+
+    output[#output + 1] = '}'
+    output[#output + 1] = '-- stylua: ignore end'
+  end
+
+  output[#output + 1] = ''
+  output[#output + 1] = 'return protocol'
+
+  local fname = './runtime/lua/vim/lsp/protocol.lua'
+  local bufnr = vim.fn.bufadd(fname)
+  vim.fn.bufload(bufnr)
+  vim.api.nvim_set_current_buf(bufnr)
+  local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false)
+  local index = vim.iter(ipairs(lines)):find(function(key, item)
+    return vim.startswith(item, '-- Generated by') and key or nil
+  end)
+  index = index and index - 1 or vim.api.nvim_buf_line_count(bufnr) - 1
+  vim.api.nvim_buf_set_lines(bufnr, index, -1, true, output)
+  vim.cmd.write()
+end
+
+---@class vim._gen_lsp.opt
+---@field output_file string
+---@field version string
+---@field methods boolean
+---@field capabilities boolean
+
+---@param opt vim._gen_lsp.opt
+function M.gen(opt)
+  --- @type vim._gen_lsp.Protocol
+  local protocol = read_json(opt)
+
+  write_to_protocol(protocol, opt.methods, opt.capabilities)
+
+  local output = {
+    '--' .. '[[',
+    'THIS FILE IS GENERATED by scr/gen/gen_lsp.lua',
+    'DO NOT EDIT MANUALLY',
+    '',
+    'Based on LSP protocol ' .. opt.version,
+    '',
+    'Regenerate:',
+    ([=[nvim -l scr/gen/gen_lsp.lua gen --version %s]=]):format(DEFAULT_LSP_VERSION),
+    '--' .. ']]',
+    '',
+    '---@meta',
+    "error('Cannot require a meta file')",
+    '',
+    '---@alias lsp.null nil',
+    '---@alias uinteger integer',
+    '---@alias decimal number',
+    '---@alias lsp.DocumentUri string',
+    '---@alias lsp.URI string',
+    '',
+  }
+
+  local anonymous_num = 0
+
+  ---@type string[]
+  local anonym_classes = {}
+
+  local simple_types = {
+    'string',
+    'boolean',
+    'integer',
+    'uinteger',
+    'decimal',
+  }
+
+  ---@param documentation string
+  local _process_documentation = function(documentation)
+    documentation = documentation:gsub('\n', '\n---')
+    -- Remove <200b> (zero-width space) unicode characters: e.g., `**/<200b>*`
+    documentation = documentation:gsub('\226\128\139', '')
+    -- Escape annotations that are not recognized by lua-ls
+    documentation = documentation:gsub('%^---@sample', '---\\@sample')
+    return '---' .. documentation
+  end
+
+  --- @class vim._gen_lsp.Type
+  --- @field kind string a common field for all Types.
+  --- @field name? string for ReferenceType, BaseType
+  --- @field element? any for ArrayType
+  --- @field items? vim._gen_lsp.Type[] for OrType, AndType
+  --- @field key? vim._gen_lsp.Type for MapType
+  --- @field value? string|vim._gen_lsp.Type for StringLiteralType, MapType, StructureLiteralType
+
+  ---@param type vim._gen_lsp.Type
+  ---@param prefix? string Optional prefix associated with the this type, made of (nested) field name.
+  ---             Used to generate class name for structure literal types.
+  ---@return string
+  local function parse_type(type, prefix)
+    -- ReferenceType | BaseType
+    if type.kind == 'reference' or type.kind == 'base' then
+      if vim.tbl_contains(simple_types, type.name) then
+        return type.name
+      end
+      return 'lsp.' .. type.name
+
+    -- ArrayType
+    elseif type.kind == 'array' then
+      local parsed_items = parse_type(type.element, prefix)
+      if type.element.items and #type.element.items > 1 then
+        parsed_items = '(' .. parsed_items .. ')'
+      end
+      return parsed_items .. '[]'
+
+    -- OrType
+    elseif type.kind == 'or' then
+      local val = ''
+      for _, item in ipairs(type.items) do
+        val = val .. parse_type(item, prefix) .. '|' --[[ @as string ]]
+      end
+      val = val:sub(0, -2)
+      return val
+
+    -- StringLiteralType
+    elseif type.kind == 'stringLiteral' then
+      return '"' .. type.value .. '"'
+
+    -- MapType
+    elseif type.kind == 'map' then
+      local key = assert(type.key)
+      local value = type.value --[[ @as vim._gen_lsp.Type ]]
+      return 'table<' .. parse_type(key, prefix) .. ', ' .. parse_type(value, prefix) .. '>'
+
+    -- StructureLiteralType
+    elseif type.kind == 'literal' then
+      -- can I use ---@param disabled? {reason: string}
+      -- use | to continue the inline class to be able to add docs
+      -- https://github.com/LuaLS/lua-language-server/issues/2128
+      anonymous_num = anonymous_num + 1
+      local anonymous_classname = 'lsp._anonym' .. anonymous_num
+      if prefix then
+        anonymous_classname = anonymous_classname .. '.' .. prefix
+      end
+      local anonym = vim
+        .iter({
+          (anonymous_num > 1 and { '' } or {}),
+          { '---@class ' .. anonymous_classname },
+        })
+        :flatten()
+        :totable()
+
+      --- @class vim._gen_lsp.StructureLiteral translated to anonymous @class.
+      --- @field deprecated? string
+      --- @field description? string
+      --- @field properties vim._gen_lsp.Property[]
+      --- @field proposed? boolean
+      --- @field since? string
+
+      ---@type vim._gen_lsp.StructureLiteral
+      local structural_literal = assert(type.value) --[[ @as vim._gen_lsp.StructureLiteral ]]
+      for _, field in ipairs(structural_literal.properties) do
+        anonym[#anonym + 1] = '---'
+        if field.documentation then
+          anonym[#anonym + 1] = _process_documentation(field.documentation)
+        end
+        anonym[#anonym + 1] = '---@field '
+          .. field.name
+          .. (field.optional and '?' or '')
+          .. ' '
+          .. parse_type(field.type, prefix .. '.' .. field.name)
+      end
+      -- anonym[#anonym + 1] = ''
+      for _, line in ipairs(anonym) do
+        if line then
+          anonym_classes[#anonym_classes + 1] = line
+        end
+      end
+      return anonymous_classname
+
+    -- TupleType
+    elseif type.kind == 'tuple' then
+      local tuple = '['
+      for _, value in ipairs(type.items) do
+        tuple = tuple .. parse_type(value, prefix) .. ', '
+      end
+      -- remove , at the end
+      tuple = tuple:sub(0, -3)
+      return tuple .. ']'
+    end
+
+    vim.print('WARNING: Unknown type ', type)
+    return ''
+  end
+
+  --- @class vim._gen_lsp.Structure translated to @class
+  --- @field deprecated? string
+  --- @field documentation? string
+  --- @field extends? { kind: string, name: string }[]
+  --- @field mixins? { kind: string, name: string }[]
+  --- @field name string
+  --- @field properties? vim._gen_lsp.Property[]  members, translated to @field
+  --- @field proposed? boolean
+  --- @field since? string
+  for _, structure in ipairs(protocol.structures) do
+    -- output[#output + 1] = ''
+    if structure.documentation then
+      output[#output + 1] = _process_documentation(structure.documentation)
+    end
+    local class_string = ('---@class lsp.%s'):format(structure.name)
+    if structure.extends or structure.mixins then
+      local inherits_from = table.concat(
+        vim.list_extend(
+          vim.tbl_map(parse_type, structure.extends or {}),
+          vim.tbl_map(parse_type, structure.mixins or {})
+        ),
+        ', '
+      )
+      class_string = class_string .. ': ' .. inherits_from
+    end
+    output[#output + 1] = class_string
+
+    --- @class vim._gen_lsp.Property translated to @field
+    --- @field deprecated? string
+    --- @field documentation? string
+    --- @field name string
+    --- @field optional? boolean
+    --- @field proposed? boolean
+    --- @field since? string
+    --- @field type { kind: string, name: string }
+    for _, field in ipairs(structure.properties or {}) do
+      output[#output + 1] = '---' -- Insert a single newline between @fields (and after @class)
+      if field.documentation then
+        output[#output + 1] = _process_documentation(field.documentation)
+      end
+      output[#output + 1] = '---@field '
+        .. field.name
+        .. (field.optional and '?' or '')
+        .. ' '
+        .. parse_type(field.type, field.name)
+    end
+    output[#output + 1] = ''
+  end
+
+  --- @class vim._gen_lsp.Enumeration translated to @enum
+  --- @field deprecated string?
+  --- @field documentation string?
+  --- @field name string?
+  --- @field proposed boolean?
+  --- @field since string?
+  --- @field suportsCustomValues boolean?
+  --- @field values { name: string, value: string, documentation?: string, since?: string }[]
+  for _, enum in ipairs(protocol.enumerations) do
+    if enum.documentation then
+      output[#output + 1] = _process_documentation(enum.documentation)
+    end
+    local enum_type = '---@alias lsp.' .. enum.name
+    for _, value in ipairs(enum.values) do
+      enum_type = enum_type
+        .. '\n---| '
+        .. (type(value.value) == 'string' and '"' .. value.value .. '"' or value.value)
+        .. ' # '
+        .. value.name
+    end
+    output[#output + 1] = enum_type
+    output[#output + 1] = ''
+  end
+
+  --- @class vim._gen_lsp.TypeAlias translated to @alias
+  --- @field deprecated? string?
+  --- @field documentation? string
+  --- @field name string
+  --- @field proposed? boolean
+  --- @field since? string
+  --- @field type vim._gen_lsp.Type
+  for _, alias in ipairs(protocol.typeAliases) do
+    if alias.documentation then
+      output[#output + 1] = _process_documentation(alias.documentation)
+    end
+    if alias.type.kind == 'or' then
+      local alias_type = '---@alias lsp.' .. alias.name .. ' '
+      for _, item in ipairs(alias.type.items) do
+        alias_type = alias_type .. parse_type(item, alias.name) .. '|'
+      end
+      alias_type = alias_type:sub(0, -2)
+      output[#output + 1] = alias_type
+    else
+      output[#output + 1] = '---@alias lsp.'
+        .. alias.name
+        .. ' '
+        .. parse_type(alias.type, alias.name)
+    end
+    output[#output + 1] = ''
+  end
+
+  -- anonymous classes
+  for _, line in ipairs(anonym_classes) do
+    output[#output + 1] = line
+  end
+
+  tofile(opt.output_file, table.concat(output, '\n') .. '\n')
+end
+
+---@type vim._gen_lsp.opt
+local opt = {
+  output_file = 'runtime/lua/vim/lsp/_meta/protocol.lua',
+  version = DEFAULT_LSP_VERSION,
+  methods = false,
+  capabilities = false,
+}
+
+local command = nil
+local i = 1
+while i <= #_G.arg do
+  if _G.arg[i] == '--out' then
+    opt.output_file = assert(_G.arg[i + 1], '--out <outfile> needed')
+    i = i + 1
+  elseif _G.arg[i] == '--version' then
+    opt.version = assert(_G.arg[i + 1], '--version <version> needed')
+    i = i + 1
+  elseif _G.arg[i] == '--methods' then
+    opt.methods = true
+  elseif _G.arg[i] == '--capabilities' then
+    opt.capabilities = true
+  elseif vim.startswith(_G.arg[i], '-') then
+    error('Unrecognized args: ' .. _G.arg[i])
+  else
+    if command then
+      error('More than one command was given: ' .. _G.arg[i])
+    else
+      command = _G.arg[i]
+    end
+  end
+  i = i + 1
+end
+
+if not command then
+  print(USAGE)
+elseif M[command] then
+  M[command](opt) -- see M.gen()
+else
+  error('Unknown command: ' .. command)
+end
+
+return M

src/gen/gen_options.lua

@@ -0,0 +1,535 @@
+--- @module 'nvim.options'
+local options = require('nvim.options')
+local options_meta = options.options
+local cstr = options.cstr
+local valid_scopes = options.valid_scopes
+
+--- @param o vim.option_meta
+--- @return string
+local function get_values_var(o)
+  return ('opt_%s_values'):format(o.abbreviation or o.full_name)
+end
+
+--- @param s string
+--- @return string
+local function lowercase_to_titlecase(s)
+  return table.concat(vim.tbl_map(function(word) --- @param word string
+    return word:sub(1, 1):upper() .. word:sub(2)
+  end, vim.split(s, '[-_]')))
+end
+
+--- @param scope string
+--- @param option_name string
+--- @return string
+local function get_scope_option(scope, option_name)
+  return ('k%sOpt%s'):format(lowercase_to_titlecase(scope), lowercase_to_titlecase(option_name))
+end
+
+local redraw_flags = {
+  ui_option = 'kOptFlagUIOption',
+  tabline = 'kOptFlagRedrTabl',
+  statuslines = 'kOptFlagRedrStat',
+  current_window = 'kOptFlagRedrWin',
+  current_buffer = 'kOptFlagRedrBuf',
+  all_windows = 'kOptFlagRedrAll',
+  curswant = 'kOptFlagCurswant',
+  highlight_only = 'kOptFlagHLOnly',
+}
+
+local list_flags = {
+  comma = 'kOptFlagComma',
+  onecomma = 'kOptFlagOneComma',
+  commacolon = 'kOptFlagComma|kOptFlagColon',
+  onecommacolon = 'kOptFlagOneComma|kOptFlagColon',
+  flags = 'kOptFlagFlagList',
+  flagscomma = 'kOptFlagComma|kOptFlagFlagList',
+}
+
+--- @param o vim.option_meta
+--- @return string
+local function get_flags(o)
+  --- @type string[]
+  local flags = { '0' }
+
+  --- @param f string
+  local function add_flag(f)
+    table.insert(flags, f)
+  end
+
+  if o.list then
+    add_flag(list_flags[o.list])
+  end
+
+  for _, r_flag in ipairs(o.redraw or {}) do
+    add_flag(redraw_flags[r_flag])
+  end
+
+  if o.expand then
+    add_flag('kOptFlagExpand')
+    if o.expand == 'nodefault' then
+      add_flag('kOptFlagNoDefExp')
+    end
+  end
+
+  for _, flag_desc in ipairs({
+    { 'nodefault', 'NoDefault' },
+    { 'no_mkrc', 'NoMkrc' },
+    { 'secure' },
+    { 'gettext' },
+    { 'noglob', 'NoGlob' },
+    { 'normal_fname_chars', 'NFname' },
+    { 'normal_dname_chars', 'NDname' },
+    { 'pri_mkrc', 'PriMkrc' },
+    { 'deny_in_modelines', 'NoML' },
+    { 'deny_duplicates', 'NoDup' },
+    { 'modelineexpr', 'MLE' },
+    { 'func' },
+  }) do
+    local key_name, flag_suffix = flag_desc[1], flag_desc[2]
+    if o[key_name] then
+      local def_name = 'kOptFlag' .. (flag_suffix or lowercase_to_titlecase(key_name))
+      add_flag(def_name)
+    end
+  end
+
+  return table.concat(flags, '|')
+end
+
+--- @param opt_type vim.option_type
+--- @return string
+local function opt_type_enum(opt_type)
+  return ('kOptValType%s'):format(lowercase_to_titlecase(opt_type))
+end
+
+--- @param scope vim.option_scope
+--- @return string
+local function opt_scope_enum(scope)
+  return ('kOptScope%s'):format(lowercase_to_titlecase(scope))
+end
+
+--- @param o vim.option_meta
+--- @return string
+local function get_scope_flags(o)
+  local scope_flags = '0'
+
+  for _, scope in ipairs(o.scope) do
+    scope_flags = ('%s | (1 << %s)'):format(scope_flags, opt_scope_enum(scope))
+  end
+
+  return scope_flags
+end
+
+--- @param o vim.option_meta
+--- @return string
+local function get_scope_idx(o)
+  --- @type string[]
+  local strs = {}
+
+  for _, scope in pairs(valid_scopes) do
+    local has_scope = vim.tbl_contains(o.scope, scope)
+    strs[#strs + 1] = ('      [%s] = %s'):format(
+      opt_scope_enum(scope),
+      get_scope_option(scope, has_scope and o.full_name or 'Invalid')
+    )
+  end
+
+  return ('{\n%s\n    }'):format(table.concat(strs, ',\n'))
+end
+
+--- @param s string
+--- @return string
+local function static_cstr_as_string(s)
+  return ('{ .data = %s, .size = sizeof(%s) - 1 }'):format(s, s)
+end
+
+--- @param v vim.option_value|function
+--- @return string
+local function get_opt_val(v)
+  --- @type vim.option_type
+  local v_type
+
+  if type(v) == 'function' then
+    v, v_type = v() --[[ @as string, vim.option_type ]]
+
+    if v_type == 'string' then
+      v = static_cstr_as_string(v)
+    end
+  else
+    v_type = type(v) --[[ @as vim.option_type ]]
+
+    if v_type == 'boolean' then
+      v = v and 'true' or 'false'
+    elseif v_type == 'number' then
+      v = ('%iL'):format(v)
+    elseif v_type == 'string' then
+      --- @cast v string
+      v = static_cstr_as_string(cstr(v))
+    end
+  end
+
+  return ('{ .type = %s, .data.%s = %s }'):format(opt_type_enum(v_type), v_type, v)
+end
+
+--- @param d vim.option_value|function
+--- @param n string
+--- @return string
+local function get_defaults(d, n)
+  if d == nil then
+    error("option '" .. n .. "' should have a default value")
+  end
+  return get_opt_val(d)
+end
+
+--- @param i integer
+--- @param o vim.option_meta
+--- @param write fun(...: string)
+local function dump_option(i, o, write)
+  write('  [', ('%u'):format(i - 1) .. ']={')
+  write('    .fullname=', cstr(o.full_name))
+  if o.abbreviation then
+    write('    .shortname=', cstr(o.abbreviation))
+  end
+  write('    .type=', opt_type_enum(o.type))
+  write('    .flags=', get_flags(o))
+  write('    .scope_flags=', get_scope_flags(o))
+  write('    .scope_idx=', get_scope_idx(o))
+  write('    .values=', (o.values and get_values_var(o) or 'NULL'))
+  write('    .values_len=', (o.values and #o.values or '0'))
+  write('    .flags_var=', (o.flags_varname and ('&%s'):format(o.flags_varname) or 'NULL'))
+  if o.enable_if then
+    write(('#if defined(%s)'):format(o.enable_if))
+  end
+
+  local is_window_local = #o.scope == 1 and o.scope[1] == 'win'
+
+  if is_window_local then
+    write('    .var=NULL')
+  elseif o.varname then
+    write('    .var=&', o.varname)
+  elseif o.immutable then
+    -- Immutable options can directly point to the default value.
+    write(('    .var=&options[%u].def_val.data'):format(i - 1))
+  else
+    error('Option must be immutable or have a variable.')
+  end
+
+  write('    .immutable=', (o.immutable and 'true' or 'false'))
+  write('    .opt_did_set_cb=', o.cb or 'NULL')
+  write('    .opt_expand_cb=', o.expand_cb or 'NULL')
+
+  if o.enable_if then
+    write('#else')
+    -- Hidden option directly points to default value.
+    write(('    .var=&options[%u].def_val.data'):format(i - 1))
+    -- Option is always immutable on the false branch of `enable_if`.
+    write('    .immutable=true')
+    write('#endif')
+  end
+
+  if not o.defaults then
+    write('    .def_val=NIL_OPTVAL')
+  elseif o.defaults.condition then
+    write(('#if defined(%s)'):format(o.defaults.condition))
+    write('    .def_val=', get_defaults(o.defaults.if_true, o.full_name))
+    if o.defaults.if_false then
+      write('#else')
+      write('    .def_val=', get_defaults(o.defaults.if_false, o.full_name))
+    end
+    write('#endif')
+  else
+    write('    .def_val=', get_defaults(o.defaults.if_true, o.full_name))
+  end
+
+  write('  },')
+end
+
+--- @param prefix string
+--- @param values vim.option_valid_values
+local function preorder_traversal(prefix, values)
+  local out = {} --- @type string[]
+
+  local function add(s)
+    table.insert(out, s)
+  end
+
+  add('')
+  add(('EXTERN const char *(%s_values[%s]) INIT( = {'):format(prefix, #vim.tbl_keys(values) + 1))
+
+  --- @type [string,vim.option_valid_values][]
+  local children = {}
+
+  for _, value in ipairs(values) do
+    if type(value) == 'string' then
+      add(('  "%s",'):format(value))
+    else
+      assert(type(value) == 'table' and type(value[1]) == 'string' and type(value[2]) == 'table')
+      add(('  "%s",'):format(value[1]))
+      table.insert(children, value)
+    end
+  end
+
+  add('  NULL')
+  add('});')
+
+  for _, value in pairs(children) do
+    -- Remove trailing colon from the added prefix to prevent syntax errors.
+    add(preorder_traversal(prefix .. '_' .. value[1]:gsub(':$', ''), value[2]))
+  end
+
+  return table.concat(out, '\n')
+end
+
+--- @param o vim.option_meta
+--- @return string
+local function gen_opt_enum(o)
+  local out = {} --- @type string[]
+
+  local function add(s)
+    table.insert(out, s)
+  end
+
+  add('')
+  add('typedef enum {')
+
+  local opt_name = lowercase_to_titlecase(o.abbreviation or o.full_name)
+  --- @type table<string,integer>
+  local enum_values
+
+  if type(o.flags) == 'table' then
+    enum_values = o.flags --[[ @as table<string,integer> ]]
+  else
+    enum_values = {}
+    for i, flag_name in ipairs(o.values) do
+      assert(type(flag_name) == 'string')
+      enum_values[flag_name] = math.pow(2, i - 1)
+    end
+  end
+
+  -- Sort the keys by the flag value so that the enum can be generated in order.
+  --- @type string[]
+  local flag_names = vim.tbl_keys(enum_values)
+  table.sort(flag_names, function(a, b)
+    return enum_values[a] < enum_values[b]
+  end)
+
+  for _, flag_name in pairs(flag_names) do
+    add(
+      ('  kOpt%sFlag%s = 0x%02x,'):format(
+        opt_name,
+        lowercase_to_titlecase(flag_name:gsub(':$', '')),
+        enum_values[flag_name]
+      )
+    )
+  end
+
+  add(('} Opt%sFlags;'):format(opt_name))
+
+  return table.concat(out, '\n')
+end
+
+--- @param output_file string
+--- @return table<string,string> options_index Map of option name to option index
+local function gen_enums(output_file)
+  --- Options for each scope.
+  --- @type table<string, vim.option_meta[]>
+  local scope_options = {}
+  for _, scope in ipairs(valid_scopes) do
+    scope_options[scope] = {}
+  end
+
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  -- Generate options enum file
+  write('// IWYU pragma: private, include "nvim/option_defs.h"')
+  write('')
+
+  --- Map of option name to option index
+  --- @type table<string, string>
+  local option_index = {}
+
+  -- Generate option index enum and populate the `option_index` and `scope_option` dicts.
+  write('typedef enum {')
+  write('  kOptInvalid = -1,')
+
+  for i, o in ipairs(options_meta) do
+    local enum_val_name = 'kOpt' .. lowercase_to_titlecase(o.full_name)
+    write(('  %s = %u,'):format(enum_val_name, i - 1))
+
+    option_index[o.full_name] = enum_val_name
+
+    if o.abbreviation then
+      option_index[o.abbreviation] = enum_val_name
+    end
+
+    local alias = o.alias or {} --[[@as string[] ]]
+    for _, v in ipairs(alias) do
+      option_index[v] = enum_val_name
+    end
+
+    for _, scope in ipairs(o.scope) do
+      table.insert(scope_options[scope], o)
+    end
+  end
+
+  write('  // Option count')
+  write('#define kOptCount ' .. tostring(#options_meta))
+  write('} OptIndex;')
+
+  -- Generate option index enum for each scope
+  for _, scope in ipairs(valid_scopes) do
+    write('')
+
+    local scope_name = lowercase_to_titlecase(scope)
+    write('typedef enum {')
+    write(('  %s = -1,'):format(get_scope_option(scope, 'Invalid')))
+
+    for idx, option in ipairs(scope_options[scope]) do
+      write(('  %s = %u,'):format(get_scope_option(scope, option.full_name), idx - 1))
+    end
+
+    write(('  // %s option count'):format(scope_name))
+    write(('#define %s %d'):format(get_scope_option(scope, 'Count'), #scope_options[scope]))
+    write(('} %sOptIndex;'):format(scope_name))
+  end
+
+  -- Generate reverse lookup from option scope index to option index for each scope.
+  for _, scope in ipairs(valid_scopes) do
+    write('')
+    write(('EXTERN const OptIndex %s_opt_idx[] INIT( = {'):format(scope))
+    for _, option in ipairs(scope_options[scope]) do
+      local idx = option_index[option.full_name]
+      write(('  [%s] = %s,'):format(get_scope_option(scope, option.full_name), idx))
+    end
+    write('});')
+  end
+
+  fd:close()
+
+  return option_index
+end
+
+--- @param output_file string
+--- @param option_index table<string,string>
+local function gen_map(output_file, option_index)
+  -- Generate option index map.
+  local hashy = require('gen.hashy')
+
+  local neworder, hashfun = hashy.hashy_hash(
+    'find_option',
+    vim.tbl_keys(option_index),
+    function(idx)
+      return ('option_hash_elems[%s].name'):format(idx)
+    end
+  )
+
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  write('static const struct { const char *name; OptIndex opt_idx; } option_hash_elems[] = {')
+
+  for _, name in ipairs(neworder) do
+    assert(option_index[name] ~= nil)
+    write(('  { .name = "%s", .opt_idx = %s },'):format(name, option_index[name]))
+  end
+
+  write('};')
+  write('')
+  write('static ' .. hashfun)
+
+  fd:close()
+end
+
+--- @param output_file string
+local function gen_vars(output_file)
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  write('// IWYU pragma: private, include "nvim/option_vars.h"')
+
+  -- Generate enums for option flags.
+  for _, o in ipairs(options_meta) do
+    if o.flags and (type(o.flags) == 'table' or o.values) then
+      write(gen_opt_enum(o))
+    end
+  end
+
+  -- Generate valid values for each option.
+  for _, option in ipairs(options_meta) do
+    -- Since option values can be nested, we need to do preorder traversal to generate the values.
+    if option.values then
+      local values_var = ('opt_%s'):format(option.abbreviation or option.full_name)
+      write(preorder_traversal(values_var, option.values))
+    end
+  end
+
+  fd:close()
+end
+
+--- @param output_file string
+local function gen_options(output_file)
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param ... string
+  local function write(...)
+    local s = table.concat({ ... }, '')
+    fd:write(s)
+    if s:match('^    %.') then
+      fd:write(',')
+    end
+    fd:write('\n')
+  end
+
+  -- Generate options[] array.
+  write([[
+  #include "nvim/ex_docmd.h"
+  #include "nvim/ex_getln.h"
+  #include "nvim/insexpand.h"
+  #include "nvim/mapping.h"
+  #include "nvim/ops.h"
+  #include "nvim/option.h"
+  #include "nvim/optionstr.h"
+  #include "nvim/quickfix.h"
+  #include "nvim/runtime.h"
+  #include "nvim/tag.h"
+  #include "nvim/window.h"
+
+  static vimoption_T options[] = {]])
+
+  for i, o in ipairs(options_meta) do
+    dump_option(i, o, write)
+  end
+
+  write('};')
+
+  fd:close()
+end
+
+local function main()
+  local options_file = arg[1]
+  local options_enum_file = arg[2]
+  local options_map_file = arg[3]
+  local option_vars_file = arg[4]
+
+  local option_index = gen_enums(options_enum_file)
+  gen_map(options_map_file, option_index)
+  gen_vars(option_vars_file)
+  gen_options(options_file)
+end
+
+main()

src/gen/gen_vimvim.lua

@@ -0,0 +1,156 @@
+local mpack = vim.mpack
+
+local syntax_file = arg[1]
+local funcs_file = arg[2]
+
+local lld = {}
+local syn_fd = io.open(syntax_file, 'w')
+lld.line_length = 0
+local function w(s)
+  syn_fd:write(s)
+  if s:find('\n') then
+    lld.line_length = #(s:gsub('.*\n', ''))
+  else
+    lld.line_length = lld.line_length + #s
+  end
+end
+
+local options = require('nvim.options')
+local auevents = require('nvim.auevents')
+local ex_cmds = require('nvim.ex_cmds')
+
+local function cmd_kw(prev_cmd, cmd)
+  if not prev_cmd then
+    return cmd:sub(1, 1) .. '[' .. cmd:sub(2) .. ']'
+  else
+    local shift = 1
+    while cmd:sub(shift, shift) == prev_cmd:sub(shift, shift) do
+      shift = shift + 1
+    end
+    if cmd:sub(1, shift) == 'def' then
+      shift = shift + 1
+    end
+    if shift >= #cmd then
+      return cmd
+    else
+      return cmd:sub(1, shift) .. '[' .. cmd:sub(shift + 1) .. ']'
+    end
+  end
+end
+
+-- Exclude these from the vimCommand keyword list, they are handled specially
+-- in syntax/vim.vim (vimAugroupKey, vimAutoCmd, vimGlobal, vimSubst). #9327
+local function is_special_cased_cmd(cmd)
+  return (
+    cmd == 'augroup'
+    or cmd == 'autocmd'
+    or cmd == 'doautocmd'
+    or cmd == 'doautoall'
+    or cmd == 'global'
+    or cmd == 'substitute'
+  )
+end
+
+local vimcmd_start = 'syn keyword vimCommand contained '
+local vimcmd_end = ' nextgroup=vimBang'
+w(vimcmd_start)
+
+local prev_cmd = nil
+for _, cmd_desc in ipairs(ex_cmds.cmds) do
+  if lld.line_length > 850 then
+    w(vimcmd_end .. '\n' .. vimcmd_start)
+  end
+  local cmd = cmd_desc.command
+  if cmd:match('%w') and cmd ~= 'z' and not is_special_cased_cmd(cmd) then
+    w(' ' .. cmd_kw(prev_cmd, cmd))
+  end
+  if cmd == 'delete' then
+    -- Add special abbreviations of :delete
+    w(' ' .. cmd_kw('d', 'dl'))
+    w(' ' .. cmd_kw('del', 'dell'))
+    w(' ' .. cmd_kw('dele', 'delel'))
+    w(' ' .. cmd_kw('delet', 'deletl'))
+    w(' ' .. cmd_kw('delete', 'deletel'))
+    w(' ' .. cmd_kw('d', 'dp'))
+    w(' ' .. cmd_kw('de', 'dep'))
+    w(' ' .. cmd_kw('del', 'delp'))
+    w(' ' .. cmd_kw('dele', 'delep'))
+    w(' ' .. cmd_kw('delet', 'deletp'))
+    w(' ' .. cmd_kw('delete', 'deletep'))
+  end
+  prev_cmd = cmd
+end
+
+w(vimcmd_end .. '\n')
+
+local vimopt_start = 'syn keyword vimOption contained '
+local vimopt_end = ' skipwhite nextgroup=vimSetEqual,vimSetMod'
+w('\n' .. vimopt_start)
+
+for _, opt_desc in ipairs(options.options) do
+  if not opt_desc.immutable then
+    if lld.line_length > 850 then
+      w(vimopt_end .. '\n' .. vimopt_start)
+    end
+    w(' ' .. opt_desc.full_name)
+    if opt_desc.abbreviation then
+      w(' ' .. opt_desc.abbreviation)
+    end
+    if opt_desc.type == 'boolean' then
+      w(' inv' .. opt_desc.full_name)
+      w(' no' .. opt_desc.full_name)
+      if opt_desc.abbreviation then
+        w(' inv' .. opt_desc.abbreviation)
+        w(' no' .. opt_desc.abbreviation)
+      end
+    end
+  end
+end
+
+w(vimopt_end .. '\n')
+
+w('\nsyn case ignore')
+local vimau_start = 'syn keyword vimAutoEvent contained '
+w('\n\n' .. vimau_start)
+
+for _, au in ipairs(auevents.events) do
+  if not auevents.nvim_specific[au[1]] then
+    if lld.line_length > 850 then
+      w('\n' .. vimau_start)
+    end
+    w(' ' .. au[1])
+    for _, alias in ipairs(au[2]) do
+      if lld.line_length > 850 then
+        w('\n' .. vimau_start)
+      end
+      -- au[1] is aliased to alias
+      w(' ' .. alias)
+    end
+  end
+end
+
+local nvimau_start = 'syn keyword nvimAutoEvent contained '
+w('\n\n' .. nvimau_start)
+
+for au, _ in vim.spairs(auevents.nvim_specific) do
+  if lld.line_length > 850 then
+    w('\n' .. nvimau_start)
+  end
+  w(' ' .. au)
+end
+
+w('\n\nsyn case match')
+local vimfun_start = 'syn keyword vimFuncName contained '
+w('\n\n' .. vimfun_start)
+local funcs = mpack.decode(io.open(funcs_file, 'rb'):read('*all'))
+for _, name in ipairs(funcs) do
+  if name then
+    if lld.line_length > 850 then
+      w('\n' .. vimfun_start)
+    end
+    w(' ' .. name)
+  end
+end
+
+w('\n')
+syn_fd:close()

src/gen/hashy.lua

@@ -0,0 +1,145 @@
+-- HASHY McHASHFACE
+
+local M = {}
+_G.d = M
+
+local function setdefault(table, key)
+  local val = table[key]
+  if val == nil then
+    val = {}
+    table[key] = val
+  end
+  return val
+end
+
+function M.build_pos_hash(strings)
+  local len_buckets = {}
+  local maxlen = 0
+  for _, s in ipairs(strings) do
+    table.insert(setdefault(len_buckets, #s), s)
+    if #s > maxlen then
+      maxlen = #s
+    end
+  end
+
+  local len_pos_buckets = {}
+  local worst_buck_size = 0
+
+  for len = 1, maxlen do
+    local strs = len_buckets[len]
+    if strs then
+      -- the best position so far generates `best_bucket`
+      -- with `minsize` worst case collisions
+      local bestpos, minsize, best_bucket = nil, #strs * 2, nil
+      for pos = 1, len do
+        local try_bucket = {}
+        for _, str in ipairs(strs) do
+          local poschar = string.sub(str, pos, pos)
+          table.insert(setdefault(try_bucket, poschar), str)
+        end
+        local maxsize = 1
+        for _, pos_strs in pairs(try_bucket) do
+          maxsize = math.max(maxsize, #pos_strs)
+        end
+        if maxsize < minsize then
+          bestpos = pos
+          minsize = maxsize
+          best_bucket = try_bucket
+        end
+      end
+      len_pos_buckets[len] = { bestpos, best_bucket }
+      worst_buck_size = math.max(worst_buck_size, minsize)
+    end
+  end
+  return len_pos_buckets, maxlen, worst_buck_size
+end
+
+function M.switcher(put, tab, maxlen, worst_buck_size)
+  local neworder = {} --- @type string[]
+  put '  switch (len) {\n'
+  local bucky = worst_buck_size > 1
+  for len = 1, maxlen do
+    local vals = tab[len]
+    if vals then
+      put('    case ' .. len .. ': ')
+      local pos, posbuck = unpack(vals)
+      local keys = vim.tbl_keys(posbuck)
+      if #keys > 1 then
+        table.sort(keys)
+        put('switch (str[' .. (pos - 1) .. ']) {\n')
+        for _, c in ipairs(keys) do
+          local buck = posbuck[c]
+          local startidx = #neworder
+          vim.list_extend(neworder, buck)
+          local endidx = #neworder
+          put("      case '" .. c .. "': ")
+          if len == 1 then
+            put('return ' .. startidx .. ';\n')
+          else
+            put('low = ' .. startidx .. '; ')
+            if bucky then
+              put('high = ' .. endidx .. '; ')
+            end
+            put 'break;\n'
+          end
+        end
+        put '      default: break;\n'
+        put '    }\n    '
+      else
+        local startidx = #neworder
+        table.insert(neworder, posbuck[keys[1]][1])
+        local endidx = #neworder
+        put('low = ' .. startidx .. '; ')
+        if bucky then
+          put('high = ' .. endidx .. '; ')
+        end
+      end
+      put 'break;\n'
+    end
+  end
+  put '    default: break;\n'
+  put '  }\n'
+  return neworder
+end
+
+function M.hashy_hash(name, strings, access)
+  local stats = {}
+  local put = function(str)
+    table.insert(stats, str)
+  end
+  local len_pos_buckets, maxlen, worst_buck_size = M.build_pos_hash(strings)
+  put('int ' .. name .. '_hash(const char *str, size_t len)\n{\n')
+  if maxlen == 1 then
+    put('\n') -- nothing
+  elseif worst_buck_size > 1 then
+    put('  int low = 0, high = 0;\n')
+  else
+    put('  int low = -1;\n')
+  end
+  local neworder = M.switcher(put, len_pos_buckets, maxlen, worst_buck_size)
+  if maxlen == 1 then
+    put([[
+  return -1;
+]])
+  elseif worst_buck_size > 1 then
+    put([[
+  for (int i = low; i < high; i++) {
+    if (!memcmp(str, ]] .. access('i') .. [[, len)) {
+      return i;
+    }
+  }
+  return -1;
+]])
+  else
+    put([[
+  if (low < 0 || memcmp(str, ]] .. access('low') .. [[, len)) {
+    return -1;
+  }
+  return low;
+]])
+  end
+  put '}\n\n'
+  return neworder, table.concat(stats)
+end
+
+return M

src/gen/luacats_grammar.lua

@@ -0,0 +1,207 @@
+--[[!
+LPEG grammar for LuaCATS
+]]
+
+local lpeg = vim.lpeg
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local C, Ct, Cg = lpeg.C, lpeg.Ct, lpeg.Cg
+
+--- @param x vim.lpeg.Pattern
+local function rep(x)
+  return x ^ 0
+end
+
+--- @param x vim.lpeg.Pattern
+local function rep1(x)
+  return x ^ 1
+end
+
+--- @param x vim.lpeg.Pattern
+local function opt(x)
+  return x ^ -1
+end
+
+local ws = rep1(S(' \t'))
+local fill = opt(ws)
+local any = P(1) -- (consume one character)
+local letter = R('az', 'AZ')
+local num = R('09')
+
+--- @param x string | vim.lpeg.Pattern
+local function Pf(x)
+  return fill * P(x) * fill
+end
+
+--- @param x string | vim.lpeg.Pattern
+local function Plf(x)
+  return fill * P(x)
+end
+
+--- @param x string
+local function Sf(x)
+  return fill * S(x) * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function paren(x)
+  return Pf('(') * x * fill * P(')')
+end
+
+--- @param x vim.lpeg.Pattern
+local function parenOpt(x)
+  return paren(x) + x
+end
+
+--- @param x vim.lpeg.Pattern
+local function comma1(x)
+  return parenOpt(x * rep(Pf(',') * x))
+end
+
+--- @param x vim.lpeg.Pattern
+local function comma(x)
+  return opt(comma1(x))
+end
+
+--- @type table<string,vim.lpeg.Pattern>
+local v = setmetatable({}, {
+  __index = function(_, k)
+    return lpeg.V(k)
+  end,
+})
+
+--- @class nvim.luacats.Param
+--- @field kind 'param'
+--- @field name string
+--- @field type string
+--- @field desc? string
+
+--- @class nvim.luacats.Return
+--- @field kind 'return'
+--- @field [integer] { type: string, name?: string}
+--- @field desc? string
+
+--- @class nvim.luacats.Generic
+--- @field kind 'generic'
+--- @field name string
+--- @field type? string
+
+--- @class nvim.luacats.Class
+--- @field kind 'class'
+--- @field name string
+--- @field parent? string
+--- @field access? 'private'|'protected'|'package'
+
+--- @class nvim.luacats.Field
+--- @field kind 'field'
+--- @field name string
+--- @field type string
+--- @field desc? string
+--- @field access? 'private'|'protected'|'package'
+
+--- @class nvim.luacats.Note
+--- @field desc? string
+
+--- @alias nvim.luacats.grammar.result
+--- | nvim.luacats.Param
+--- | nvim.luacats.Return
+--- | nvim.luacats.Generic
+--- | nvim.luacats.Class
+--- | nvim.luacats.Field
+--- | nvim.luacats.Note
+
+--- @class nvim.luacats.grammar
+--- @field match fun(self, input: string): nvim.luacats.grammar.result?
+
+local function annot(nm, pat)
+  if type(nm) == 'string' then
+    nm = P(nm)
+  end
+  if pat then
+    return Ct(Cg(P(nm), 'kind') * fill * pat)
+  end
+  return Ct(Cg(P(nm), 'kind'))
+end
+
+local colon = Pf(':')
+local ellipsis = P('...')
+local ident_first = P('_') + letter
+local ident = ident_first * rep(ident_first + num)
+local opt_ident = ident * opt(P('?'))
+local ty_ident_sep = S('-._')
+local ty_ident = ident * rep(ty_ident_sep * ident)
+local string_single = P "'" * rep(any - P "'") * P "'"
+local string_double = P('"') * rep(any - P('"')) * P('"')
+local generic = P('`') * ty_ident * P('`')
+local literal = string_single + string_double + (opt(P('-')) * rep1(num)) + P('false') + P('true')
+local ty_prims = ty_ident + literal + generic
+
+local array_postfix = rep1(Plf('[]'))
+local opt_postfix = rep1(Plf('?'))
+local rep_array_opt_postfix = rep(array_postfix + opt_postfix)
+
+local typedef = P({
+  'typedef',
+  typedef = C(v.type),
+
+  type = v.ty * rep_array_opt_postfix * rep(Pf('|') * v.ty * rep_array_opt_postfix),
+  ty = v.composite + paren(v.typedef),
+  composite = (v.types * array_postfix) + (v.types * opt_postfix) + v.types,
+  types = v.generics + v.kv_table + v.tuple + v.dict + v.table_literal + v.fun + ty_prims,
+
+  tuple = Pf('[') * comma1(v.type) * Plf(']'),
+  dict = Pf('{') * comma1(Pf('[') * v.type * Pf(']') * colon * v.type) * Plf('}'),
+  kv_table = Pf('table') * Pf('<') * v.type * Pf(',') * v.type * Plf('>'),
+  table_literal = Pf('{') * comma1(opt_ident * Pf(':') * v.type) * Plf('}'),
+  fun_param = (opt_ident + ellipsis) * opt(colon * v.type),
+  fun_ret = v.type + (ellipsis * opt(colon * v.type)),
+  fun = Pf('fun') * paren(comma(v.fun_param)) * opt(Pf(':') * comma1(v.fun_ret)),
+  generics = P(ty_ident) * Pf('<') * comma1(v.type) * Plf('>'),
+}) / function(match)
+  return vim.trim(match):gsub('^%((.*)%)$', '%1'):gsub('%?+', '?')
+end
+
+local access = P('private') + P('protected') + P('package')
+local caccess = Cg(access, 'access')
+local cattr = Cg(comma(access + P('exact')), 'access')
+local desc_delim = Sf '#:' + ws
+local desc = Cg(rep(any), 'desc')
+local opt_desc = opt(desc_delim * desc)
+local ty_name = Cg(ty_ident, 'name')
+local opt_parent = opt(colon * Cg(ty_ident, 'parent'))
+local lname = (ident + ellipsis) * opt(P('?'))
+
+local grammar = P {
+  rep1(P('@') * (v.ats + v.ext_ats)),
+
+  ats = annot('param', Cg(lname, 'name') * ws * v.ctype * opt_desc)
+    + annot('return', comma1(Ct(v.ctype * opt(ws * (ty_name + Cg(ellipsis, 'name'))))) * opt_desc)
+    + annot('type', comma1(Ct(v.ctype)) * opt_desc)
+    + annot('cast', ty_name * ws * opt(Sf('+-')) * v.ctype)
+    + annot('generic', ty_name * opt(colon * v.ctype))
+    + annot('class', opt(paren(cattr)) * fill * ty_name * opt_parent)
+    + annot('field', opt(caccess * ws) * v.field_name * ws * v.ctype * opt_desc)
+    + annot('operator', ty_name * opt(paren(Cg(v.ctype, 'argtype'))) * colon * v.ctype)
+    + annot(access)
+    + annot('deprecated')
+    + annot('alias', ty_name * opt(ws * v.ctype))
+    + annot('enum', ty_name)
+    + annot('overload', v.ctype)
+    + annot('see', opt(desc_delim) * desc)
+    + annot('diagnostic', opt(desc_delim) * desc)
+    + annot('meta'),
+
+  --- Custom extensions
+  ext_ats = (
+    annot('note', desc)
+    + annot('since', desc)
+    + annot('nodoc')
+    + annot('inlinedoc')
+    + annot('brief', desc)
+  ),
+
+  field_name = Cg(lname + (v.ty_index * opt(P('?'))), 'name'),
+  ty_index = C(Pf('[') * typedef * fill * P(']')),
+  ctype = Cg(typedef, 'type'),
+}
+
+return grammar --[[@as nvim.luacats.grammar]]

src/gen/luacats_parser.lua

@@ -0,0 +1,535 @@
+local luacats_grammar = require('gen.luacats_grammar')
+
+--- @class nvim.luacats.parser.param : nvim.luacats.Param
+
+--- @class nvim.luacats.parser.return
+--- @field name string
+--- @field type string
+--- @field desc string
+
+--- @class nvim.luacats.parser.note
+--- @field desc string
+
+--- @class nvim.luacats.parser.brief
+--- @field kind 'brief'
+--- @field desc string
+
+--- @class nvim.luacats.parser.alias
+--- @field kind 'alias'
+--- @field type string[]
+--- @field desc string
+
+--- @class nvim.luacats.parser.fun
+--- @field name string
+--- @field params nvim.luacats.parser.param[]
+--- @field returns nvim.luacats.parser.return[]
+--- @field desc string
+--- @field access? 'private'|'package'|'protected'
+--- @field class? string
+--- @field module? string
+--- @field modvar? string
+--- @field classvar? string
+--- @field deprecated? true
+--- @field since? string
+--- @field attrs? string[]
+--- @field nodoc? true
+--- @field generics? table<string,string>
+--- @field table? true
+--- @field notes? nvim.luacats.parser.note[]
+--- @field see? nvim.luacats.parser.note[]
+
+--- @class nvim.luacats.parser.field : nvim.luacats.Field
+--- @field classvar? string
+--- @field nodoc? true
+
+--- @class nvim.luacats.parser.class : nvim.luacats.Class
+--- @field desc? string
+--- @field nodoc? true
+--- @field inlinedoc? true
+--- @field fields nvim.luacats.parser.field[]
+--- @field notes? string[]
+
+--- @class nvim.luacats.parser.State
+--- @field doc_lines? string[]
+--- @field cur_obj? nvim.luacats.parser.obj
+--- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note
+--- @field last_doc_item_indent? integer
+
+--- @alias nvim.luacats.parser.obj
+--- | nvim.luacats.parser.class
+--- | nvim.luacats.parser.fun
+--- | nvim.luacats.parser.brief
+--- | nvim.luacats.parser.alias
+
+-- Remove this when we document classes properly
+--- Some doc lines have the form:
+---   param name some.complex.type (table) description
+--- if so then transform the line to remove the complex type:
+---   param name (table) description
+--- @param line string
+local function use_type_alt(line)
+  for _, type in ipairs({ 'table', 'function' }) do
+    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2')
+    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2')
+    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2')
+
+    line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1')
+    line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1')
+    line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1')
+  end
+  return line
+end
+
+--- If we collected any `---` lines. Add them to the existing (or new) object
+--- Used for function/class descriptions and multiline param descriptions.
+--- @param state nvim.luacats.parser.State
+local function add_doc_lines_to_obj(state)
+  if state.doc_lines then
+    state.cur_obj = state.cur_obj or {}
+    local cur_obj = assert(state.cur_obj)
+    local txt = table.concat(state.doc_lines, '\n')
+    if cur_obj.desc then
+      cur_obj.desc = cur_obj.desc .. '\n' .. txt
+    else
+      cur_obj.desc = txt
+    end
+    state.doc_lines = nil
+  end
+end
+
+--- @param line string
+--- @param state nvim.luacats.parser.State
+local function process_doc_line(line, state)
+  line = line:sub(4):gsub('^%s+@', '@')
+  line = use_type_alt(line)
+
+  local parsed = luacats_grammar:match(line)
+
+  if not parsed then
+    if line:match('^ ') then
+      line = line:sub(2)
+    end
+
+    if state.last_doc_item then
+      if not state.last_doc_item_indent then
+        state.last_doc_item_indent = #line:match('^%s*') + 1
+      end
+      state.last_doc_item.desc = (state.last_doc_item.desc or '')
+        .. '\n'
+        .. line:sub(state.last_doc_item_indent or 1)
+    else
+      state.doc_lines = state.doc_lines or {}
+      table.insert(state.doc_lines, line)
+    end
+    return
+  end
+
+  state.last_doc_item_indent = nil
+  state.last_doc_item = nil
+  state.cur_obj = state.cur_obj or {}
+  local cur_obj = assert(state.cur_obj)
+
+  local kind = parsed.kind
+
+  if kind == 'brief' then
+    state.cur_obj = {
+      kind = 'brief',
+      desc = parsed.desc,
+    }
+  elseif kind == 'class' then
+    --- @cast parsed nvim.luacats.Class
+    cur_obj.kind = 'class'
+    cur_obj.name = parsed.name
+    cur_obj.parent = parsed.parent
+    cur_obj.access = parsed.access
+    cur_obj.desc = state.doc_lines and table.concat(state.doc_lines, '\n') or nil
+    state.doc_lines = nil
+    cur_obj.fields = {}
+  elseif kind == 'field' then
+    --- @cast parsed nvim.luacats.Field
+    parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil
+    if parsed.desc then
+      parsed.desc = vim.trim(parsed.desc)
+    end
+    table.insert(cur_obj.fields, parsed)
+    state.doc_lines = nil
+  elseif kind == 'operator' then
+    parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil
+    if parsed.desc then
+      parsed.desc = vim.trim(parsed.desc)
+    end
+    table.insert(cur_obj.fields, parsed)
+    state.doc_lines = nil
+  elseif kind == 'param' then
+    state.last_doc_item_indent = nil
+    cur_obj.params = cur_obj.params or {}
+    if vim.endswith(parsed.name, '?') then
+      parsed.name = parsed.name:sub(1, -2)
+      parsed.type = parsed.type .. '?'
+    end
+    state.last_doc_item = {
+      name = parsed.name,
+      type = parsed.type,
+      desc = parsed.desc,
+    }
+    table.insert(cur_obj.params, state.last_doc_item)
+  elseif kind == 'return' then
+    cur_obj.returns = cur_obj.returns or {}
+    for _, t in ipairs(parsed) do
+      table.insert(cur_obj.returns, {
+        name = t.name,
+        type = t.type,
+        desc = parsed.desc,
+      })
+    end
+    state.last_doc_item_indent = nil
+    state.last_doc_item = cur_obj.returns[#cur_obj.returns]
+  elseif kind == 'private' then
+    cur_obj.access = 'private'
+  elseif kind == 'package' then
+    cur_obj.access = 'package'
+  elseif kind == 'protected' then
+    cur_obj.access = 'protected'
+  elseif kind == 'deprecated' then
+    cur_obj.deprecated = true
+  elseif kind == 'inlinedoc' then
+    cur_obj.inlinedoc = true
+  elseif kind == 'nodoc' then
+    cur_obj.nodoc = true
+  elseif kind == 'since' then
+    cur_obj.since = parsed.desc
+  elseif kind == 'see' then
+    cur_obj.see = cur_obj.see or {}
+    table.insert(cur_obj.see, { desc = parsed.desc })
+  elseif kind == 'note' then
+    state.last_doc_item_indent = nil
+    state.last_doc_item = {
+      desc = parsed.desc,
+    }
+    cur_obj.notes = cur_obj.notes or {}
+    table.insert(cur_obj.notes, state.last_doc_item)
+  elseif kind == 'type' then
+    cur_obj.desc = parsed.desc
+    parsed.desc = nil
+    parsed.kind = nil
+    cur_obj.type = parsed
+  elseif kind == 'alias' then
+    state.cur_obj = {
+      kind = 'alias',
+      desc = parsed.desc,
+    }
+  elseif kind == 'enum' then
+    -- TODO
+    state.doc_lines = nil
+  elseif
+    vim.tbl_contains({
+      'diagnostic',
+      'cast',
+      'overload',
+      'meta',
+    }, kind)
+  then
+    -- Ignore
+    return
+  elseif kind == 'generic' then
+    cur_obj.generics = cur_obj.generics or {}
+    cur_obj.generics[parsed.name] = parsed.type or 'any'
+  else
+    error('Unhandled' .. vim.inspect(parsed))
+  end
+end
+
+--- @param fun nvim.luacats.parser.fun
+--- @return nvim.luacats.parser.field
+local function fun2field(fun)
+  local parts = { 'fun(' }
+
+  local params = {} ---@type string[]
+  for _, p in ipairs(fun.params or {}) do
+    params[#params + 1] = string.format('%s: %s', p.name, p.type)
+  end
+  parts[#parts + 1] = table.concat(params, ', ')
+  parts[#parts + 1] = ')'
+  if fun.returns then
+    parts[#parts + 1] = ': '
+    local tys = {} --- @type string[]
+    for _, p in ipairs(fun.returns) do
+      tys[#tys + 1] = p.type
+    end
+    parts[#parts + 1] = table.concat(tys, ', ')
+  end
+
+  return {
+    name = fun.name,
+    type = table.concat(parts, ''),
+    access = fun.access,
+    desc = fun.desc,
+    nodoc = fun.nodoc,
+  }
+end
+
+--- Function to normalize known form for declaring functions and normalize into a more standard
+--- form.
+--- @param line string
+--- @return string
+local function filter_decl(line)
+  -- M.fun = vim._memoize(function(...)
+  --   ->
+  -- function M.fun(...)
+  line = line:gsub('^local (.+) = memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)')
+  line = line:gsub('^(.+) = memoize%([^,]+, function%((.*)%)$', 'function %1(%2)')
+  return line
+end
+
+--- @param line string
+--- @param state nvim.luacats.parser.State
+--- @param classes table<string,nvim.luacats.parser.class>
+--- @param classvars table<string,string>
+--- @param has_indent boolean
+local function process_lua_line(line, state, classes, classvars, has_indent)
+  line = filter_decl(line)
+
+  if state.cur_obj and state.cur_obj.kind == 'class' then
+    local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=')
+    if nm then
+      classvars[nm] = state.cur_obj.name
+    end
+    return
+  end
+
+  do
+    local parent_tbl, sep, fun_or_meth_nm =
+      line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(')
+    if parent_tbl then
+      -- Have a decl. Ensure cur_obj
+      state.cur_obj = state.cur_obj or {}
+      local cur_obj = assert(state.cur_obj)
+
+      -- Match `Class:foo` methods for defined classes
+      local class = classvars[parent_tbl]
+      if class then
+        --- @cast cur_obj nvim.luacats.parser.fun
+        cur_obj.name = fun_or_meth_nm
+        cur_obj.class = class
+        cur_obj.classvar = parent_tbl
+        -- Add self param to methods
+        if sep == ':' then
+          cur_obj.params = cur_obj.params or {}
+          table.insert(cur_obj.params, 1, {
+            name = 'self',
+            type = class,
+          })
+        end
+
+        -- Add method as the field to the class
+        local cls = classes[class]
+        local field = fun2field(cur_obj)
+        field.classvar = cur_obj.classvar
+        table.insert(cls.fields, field)
+        return
+      end
+
+      -- Match `M.foo`
+      if cur_obj and parent_tbl == cur_obj.modvar then
+        cur_obj.name = fun_or_meth_nm
+        return
+      end
+    end
+  end
+
+  do
+    -- Handle: `function A.B.C.foo(...)`
+    local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(')
+    if fn_nm then
+      state.cur_obj = state.cur_obj or {}
+      state.cur_obj.name = fn_nm
+      return
+    end
+  end
+
+  do
+    -- Handle: `M.foo = {...}` where `M` is the modvar
+    local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=')
+    if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then
+      state.cur_obj.name = tbl_nm
+      state.cur_obj.table = true
+      return
+    end
+  end
+
+  do
+    -- Handle: `foo = {...}`
+    local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=')
+    if tbl_nm and not has_indent then
+      state.cur_obj = state.cur_obj or {}
+      state.cur_obj.name = tbl_nm
+      state.cur_obj.table = true
+      return
+    end
+  end
+
+  do
+    -- Handle: `vim.foo = {...}`
+    local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=')
+    if state.cur_obj and tbl_nm and not has_indent then
+      state.cur_obj.name = tbl_nm
+      state.cur_obj.table = true
+      return
+    end
+  end
+
+  if state.cur_obj then
+    if line:find('^%s*%-%- luacheck:') then
+      state.cur_obj = nil
+    elseif line:find('^%s*local%s+') then
+      state.cur_obj = nil
+    elseif line:find('^%s*return%s+') then
+      state.cur_obj = nil
+    elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then
+      state.cur_obj = nil
+    end
+  end
+end
+
+--- Determine the table name used to export functions of a module
+--- Usually this is `M`.
+--- @param str string
+--- @return string?
+local function determine_modvar(str)
+  local modvar --- @type string?
+  for line in vim.gsplit(str, '\n') do
+    do
+      --- @type string?
+      local m = line:match('^return%s+([a-zA-Z_]+)')
+      if m then
+        modvar = m
+      end
+    end
+    do
+      --- @type string?
+      local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),')
+      if m then
+        modvar = m
+      end
+    end
+  end
+  return modvar
+end
+
+--- @param obj nvim.luacats.parser.obj
+--- @param funs nvim.luacats.parser.fun[]
+--- @param classes table<string,nvim.luacats.parser.class>
+--- @param briefs string[]
+--- @param uncommitted nvim.luacats.parser.obj[]
+local function commit_obj(obj, classes, funs, briefs, uncommitted)
+  local commit = false
+  if obj.kind == 'class' then
+    --- @cast obj nvim.luacats.parser.class
+    if not classes[obj.name] then
+      classes[obj.name] = obj
+      commit = true
+    end
+  elseif obj.kind == 'alias' then
+    -- Just pretend
+    commit = true
+  elseif obj.kind == 'brief' then
+    --- @cast obj nvim.luacats.parser.brief`
+    briefs[#briefs + 1] = obj.desc
+    commit = true
+  else
+    --- @cast obj nvim.luacats.parser.fun`
+    if obj.name then
+      funs[#funs + 1] = obj
+      commit = true
+    end
+  end
+  if not commit then
+    table.insert(uncommitted, obj)
+  end
+  return commit
+end
+
+--- @param filename string
+--- @param uncommitted nvim.luacats.parser.obj[]
+-- luacheck: no unused
+local function dump_uncommitted(filename, uncommitted)
+  local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt'
+  if #uncommitted > 0 then
+    print(string.format('Could not commit %d objects in %s', #uncommitted, filename))
+    vim.fn.mkdir(vim.fs.dirname(out_path), 'p')
+    local f = assert(io.open(out_path, 'w'))
+    for i, x in ipairs(uncommitted) do
+      f:write(i)
+      f:write(': ')
+      f:write(vim.inspect(x))
+      f:write('\n')
+    end
+    f:close()
+  else
+    vim.fn.delete(out_path)
+  end
+end
+
+local M = {}
+
+function M.parse_str(str, filename)
+  local funs = {} --- @type nvim.luacats.parser.fun[]
+  local classes = {} --- @type table<string,nvim.luacats.parser.class>
+  local briefs = {} --- @type string[]
+
+  local mod_return = determine_modvar(str)
+
+  --- @type string
+  local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename
+  module = module:gsub('/', '.')
+
+  local classvars = {} --- @type table<string,string>
+
+  local state = {} --- @type nvim.luacats.parser.State
+
+  -- Keep track of any partial objects we don't commit
+  local uncommitted = {} --- @type nvim.luacats.parser.obj[]
+
+  for line in vim.gsplit(str, '\n') do
+    local has_indent = line:match('^%s+') ~= nil
+    line = vim.trim(line)
+    if vim.startswith(line, '---') then
+      process_doc_line(line, state)
+    else
+      add_doc_lines_to_obj(state)
+
+      if state.cur_obj then
+        state.cur_obj.modvar = mod_return
+        state.cur_obj.module = module
+      end
+
+      process_lua_line(line, state, classes, classvars, has_indent)
+
+      -- Commit the object
+      local cur_obj = state.cur_obj
+      if cur_obj then
+        if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then
+          --- @diagnostic disable-next-line:inject-field
+          cur_obj.line = line
+        end
+      end
+
+      state = {}
+    end
+  end
+
+  -- dump_uncommitted(filename, uncommitted)
+
+  return classes, funs, briefs, uncommitted
+end
+
+--- @param filename string
+function M.parse(filename)
+  local f = assert(io.open(filename, 'r'))
+  local txt = f:read('*all')
+  f:close()
+
+  return M.parse_str(txt, filename)
+end
+
+return M

src/gen/nvim_version.lua.in

@@ -0,0 +1,9 @@
+return {
+  {"major", ${NVIM_VERSION_MAJOR}},
+  {"minor", ${NVIM_VERSION_MINOR}},
+  {"patch", ${NVIM_VERSION_PATCH}},
+  {"prerelease", "${NVIM_VERSION_PRERELEASE}" ~= ""},
+  {"api_level", ${NVIM_API_LEVEL}},
+  {"api_compatible", ${NVIM_API_LEVEL_COMPAT}},
+  {"api_prerelease", ${NVIM_API_PRERELEASE}},
+}

src/gen/preload.lua

@@ -0,0 +1,6 @@
+local srcdir = table.remove(arg, 1)
+
+package.path = (srcdir .. '/src/?.lua;') .. (srcdir .. '/runtime/lua/?.lua;') .. package.path
+
+arg[0] = table.remove(arg, 1)
+return loadfile(arg[0])()

src/gen/preload_nlua.lua

@@ -0,0 +1,17 @@
+local srcdir = table.remove(arg, 1)
+local nlualib = table.remove(arg, 1)
+local gendir = table.remove(arg, 1)
+
+package.path = (srcdir .. '/src/?.lua;')
+  .. (srcdir .. '/runtime/lua/?.lua;')
+  .. (gendir .. '/?.lua;')
+  .. package.path
+
+_G.vim = require 'vim.shared'
+_G.vim.inspect = require 'vim.inspect'
+package.cpath = package.cpath .. ';' .. nlualib
+require 'nlua0'
+vim.NIL = vim.mpack.NIL -- WOW BOB WOW
+
+arg[0] = table.remove(arg, 1)
+return loadfile(arg[0])()

src/gen/util.lua

@@ -0,0 +1,399 @@
+-- TODO(justinmk): move most of this to `vim.text`.
+
+local fmt = string.format
+
+--- @class nvim.util.MDNode
+--- @field [integer] nvim.util.MDNode
+--- @field type string
+--- @field text? string
+
+local INDENTATION = 4
+
+local NBSP = string.char(160)
+
+local M = {}
+
+local function contains(t, xs)
+  return vim.tbl_contains(xs, t)
+end
+
+-- Map of api_level:version, by inspection of:
+--    :lua= vim.mpack.decode(vim.fn.readfile('test/functional/fixtures/api_level_9.mpack','B')).version
+M.version_level = {
+  [13] = '0.11.0',
+  [12] = '0.10.0',
+  [11] = '0.9.0',
+  [10] = '0.8.0',
+  [9] = '0.7.0',
+  [8] = '0.6.0',
+  [7] = '0.5.0',
+  [6] = '0.4.0',
+  [5] = '0.3.2',
+  [4] = '0.3.0',
+  [3] = '0.2.1',
+  [2] = '0.2.0',
+  [1] = '0.1.0',
+}
+
+--- @param txt string
+--- @param srow integer
+--- @param scol integer
+--- @param erow? integer
+--- @param ecol? integer
+--- @return string
+local function slice_text(txt, srow, scol, erow, ecol)
+  local lines = vim.split(txt, '\n')
+
+  if srow == erow then
+    return lines[srow + 1]:sub(scol + 1, ecol)
+  end
+
+  if erow then
+    -- Trim the end
+    for _ = erow + 2, #lines do
+      table.remove(lines, #lines)
+    end
+  end
+
+  -- Trim the start
+  for _ = 1, srow do
+    table.remove(lines, 1)
+  end
+
+  lines[1] = lines[1]:sub(scol + 1)
+  lines[#lines] = lines[#lines]:sub(1, ecol)
+
+  return table.concat(lines, '\n')
+end
+
+--- @param text string
+--- @return nvim.util.MDNode
+local function parse_md_inline(text)
+  local parser = vim.treesitter.languagetree.new(text, 'markdown_inline')
+  local root = parser:parse(true)[1]:root()
+
+  --- @param node TSNode
+  --- @return nvim.util.MDNode?
+  local function extract(node)
+    local ntype = node:type()
+
+    if ntype:match('^%p$') then
+      return
+    end
+
+    --- @type table<any,any>
+    local ret = { type = ntype }
+    ret.text = vim.treesitter.get_node_text(node, text)
+
+    local row, col = 0, 0
+
+    for child, child_field in node:iter_children() do
+      local e = extract(child)
+      if e and ntype == 'inline' then
+        local srow, scol = child:start()
+        if (srow == row and scol > col) or srow > row then
+          local t = slice_text(ret.text, row, col, srow, scol)
+          if t and t ~= '' then
+            table.insert(ret, { type = 'text', j = true, text = t })
+          end
+        end
+        row, col = child:end_()
+      end
+
+      if child_field then
+        ret[child_field] = e
+      else
+        table.insert(ret, e)
+      end
+    end
+
+    if ntype == 'inline' and (row > 0 or col > 0) then
+      local t = slice_text(ret.text, row, col)
+      if t and t ~= '' then
+        table.insert(ret, { type = 'text', text = t })
+      end
+    end
+
+    return ret
+  end
+
+  return extract(root) or {}
+end
+
+--- @param text string
+--- @return nvim.util.MDNode
+local function parse_md(text)
+  local parser = vim.treesitter.languagetree.new(text, 'markdown', {
+    injections = { markdown = '' },
+  })
+
+  local root = parser:parse(true)[1]:root()
+
+  local EXCLUDE_TEXT_TYPE = {
+    list = true,
+    list_item = true,
+    section = true,
+    document = true,
+    fenced_code_block = true,
+    fenced_code_block_delimiter = true,
+  }
+
+  --- @param node TSNode
+  --- @return nvim.util.MDNode?
+  local function extract(node)
+    local ntype = node:type()
+
+    if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then
+      return
+    end
+
+    --- @type table<any,any>
+    local ret = { type = ntype }
+
+    if not EXCLUDE_TEXT_TYPE[ntype] then
+      ret.text = vim.treesitter.get_node_text(node, text)
+    end
+
+    if ntype == 'inline' then
+      ret = parse_md_inline(ret.text)
+    end
+
+    for child, child_field in node:iter_children() do
+      local e = extract(child)
+      if child_field then
+        ret[child_field] = e
+      else
+        table.insert(ret, e)
+      end
+    end
+
+    return ret
+  end
+
+  return extract(root) or {}
+end
+
+--- Prefixes each line in `text`.
+---
+--- Does not wrap, not important for "meta" files? (You probably want md_to_vimdoc instead.)
+---
+--- @param text string
+--- @param prefix_ string
+function M.prefix_lines(prefix_, text)
+  local r = ''
+  for _, l in ipairs(vim.split(text, '\n', { plain = true })) do
+    r = r .. vim.trim(prefix_ .. l) .. '\n'
+  end
+  return r
+end
+
+--- @param x string
+--- @param start_indent integer
+--- @param indent integer
+--- @param text_width integer
+--- @return string
+function M.wrap(x, start_indent, indent, text_width)
+  local words = vim.split(vim.trim(x), '%s+')
+  local parts = { string.rep(' ', start_indent) } --- @type string[]
+  local count = indent
+
+  for i, w in ipairs(words) do
+    if count > indent and count + #w > text_width - 1 then
+      parts[#parts + 1] = '\n'
+      parts[#parts + 1] = string.rep(' ', indent)
+      count = indent
+    elseif i ~= 1 then
+      parts[#parts + 1] = ' '
+      count = count + 1
+    end
+    count = count + #w
+    parts[#parts + 1] = w
+  end
+
+  return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', ''))
+end
+
+--- @param node nvim.util.MDNode
+--- @param start_indent integer
+--- @param indent integer
+--- @param text_width integer
+--- @param level integer
+--- @return string[]
+local function render_md(node, start_indent, indent, text_width, level, is_list)
+  local parts = {} --- @type string[]
+
+  -- For debugging
+  local add_tag = false
+  -- local add_tag = true
+
+  local ntype = node.type
+
+  if add_tag then
+    parts[#parts + 1] = '<' .. ntype .. '>'
+  end
+
+  if ntype == 'text' then
+    parts[#parts + 1] = node.text
+  elseif ntype == 'html_tag' then
+    error('html_tag: ' .. node.text)
+  elseif ntype == 'inline_link' then
+    vim.list_extend(parts, { '*', node[1].text, '*' })
+  elseif ntype == 'shortcut_link' then
+    if node[1].text:find('^<.*>$') then
+      parts[#parts + 1] = node[1].text
+    elseif node[1].text:find('^%d+$') then
+      vim.list_extend(parts, { '[', node[1].text, ']' })
+    else
+      vim.list_extend(parts, { '|', node[1].text, '|' })
+    end
+  elseif ntype == 'backslash_escape' then
+    parts[#parts + 1] = node.text
+  elseif ntype == 'emphasis' then
+    parts[#parts + 1] = node.text:sub(2, -2)
+  elseif ntype == 'code_span' then
+    vim.list_extend(parts, { '`', node.text:sub(2, -2):gsub(' ', NBSP), '`' })
+  elseif ntype == 'inline' then
+    if #node == 0 then
+      local text = assert(node.text)
+      parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width)
+    else
+      for _, child in ipairs(node) do
+        vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1))
+      end
+    end
+  elseif ntype == 'paragraph' then
+    local pparts = {}
+    for _, child in ipairs(node) do
+      vim.list_extend(pparts, render_md(child, start_indent, indent, text_width, level + 1))
+    end
+    parts[#parts + 1] = M.wrap(table.concat(pparts), start_indent, indent, text_width)
+    parts[#parts + 1] = '\n'
+  elseif ntype == 'code_fence_content' then
+    local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n')
+
+    local cindent = indent + INDENTATION
+    if level > 3 then
+      -- The tree-sitter markdown parser doesn't parse the code blocks indents
+      -- correctly in lists. Fudge it!
+      lines[1] = '    ' .. lines[1] -- ¯\_(ツ)_/¯
+      cindent = indent - level
+      local _, initial_indent = lines[1]:find('^%s*')
+      initial_indent = initial_indent + cindent
+      if initial_indent < indent then
+        cindent = indent - INDENTATION
+      end
+    end
+
+    for _, l in ipairs(lines) do
+      if #l > 0 then
+        parts[#parts + 1] = string.rep(' ', cindent)
+        parts[#parts + 1] = l
+      end
+      parts[#parts + 1] = '\n'
+    end
+  elseif ntype == 'fenced_code_block' then
+    parts[#parts + 1] = '>'
+    for _, child in ipairs(node) do
+      if child.type == 'info_string' then
+        parts[#parts + 1] = child.text
+        break
+      end
+    end
+    parts[#parts + 1] = '\n'
+    for _, child in ipairs(node) do
+      if child.type ~= 'info_string' then
+        vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1))
+      end
+    end
+    parts[#parts + 1] = '<\n'
+  elseif ntype == 'html_block' then
+    local text = node.text:gsub('^<pre>help', '')
+    text = text:gsub('</pre>%s*$', '')
+    parts[#parts + 1] = text
+  elseif ntype == 'list_marker_dot' then
+    parts[#parts + 1] = node.text
+  elseif contains(ntype, { 'list_marker_minus', 'list_marker_star' }) then
+    parts[#parts + 1] = '• '
+  elseif ntype == 'list_item' then
+    parts[#parts + 1] = string.rep(' ', indent)
+    local offset = node[1].type == 'list_marker_dot' and 3 or 2
+    for i, child in ipairs(node) do
+      local sindent = i <= 2 and 0 or (indent + offset)
+      vim.list_extend(
+        parts,
+        render_md(child, sindent, indent + offset, text_width, level + 1, true)
+      )
+    end
+  else
+    if node.text then
+      error(fmt('cannot render:\n%s', vim.inspect(node)))
+    end
+    for i, child in ipairs(node) do
+      local start_indent0 = i == 1 and start_indent or indent
+      vim.list_extend(
+        parts,
+        render_md(child, start_indent0, indent, text_width, level + 1, is_list)
+      )
+      if ntype ~= 'list' and i ~= #node then
+        if (node[i + 1] or {}).type ~= 'list' then
+          parts[#parts + 1] = '\n'
+        end
+      end
+    end
+  end
+
+  if add_tag then
+    parts[#parts + 1] = '</' .. ntype .. '>'
+  end
+
+  return parts
+end
+
+--- @param text_width integer
+local function align_tags(text_width)
+  --- @param line string
+  --- @return string
+  return function(line)
+    local tag_pat = '%s*(%*.+%*)%s*$'
+    local tags = {}
+    for m in line:gmatch(tag_pat) do
+      table.insert(tags, m)
+    end
+
+    if #tags > 0 then
+      line = line:gsub(tag_pat, '')
+      local tags_str = ' ' .. table.concat(tags, ' ')
+      --- @type integer
+      local conceal_offset = select(2, tags_str:gsub('%*', '')) - 2
+      local pad = string.rep(' ', text_width - #line - #tags_str + conceal_offset)
+      return line .. pad .. tags_str
+    end
+
+    return line
+  end
+end
+
+--- @param text string
+--- @param start_indent integer
+--- @param indent integer
+--- @param is_list? boolean
+--- @return string
+function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list)
+  -- Add an extra newline so the parser can properly capture ending ```
+  local parsed = parse_md(text .. '\n')
+  local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list)
+
+  local lines = vim.split(table.concat(ret):gsub(NBSP, ' '), '\n')
+
+  lines = vim.tbl_map(align_tags(text_width), lines)
+
+  local s = table.concat(lines, '\n')
+
+  -- Reduce whitespace in code-blocks
+  s = s:gsub('\n+%s*>([a-z]+)\n', ' >%1\n')
+  s = s:gsub('\n+%s*>\n?\n', ' >\n')
+
+  return s
+end
+
+return M