mirrors/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2025-12-16 03:15:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(docs): replace lua2dox.lua

Browse Source 
Problem:

The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.

Solution:

Re-implement the flow using Lua, LPEG and treesitter.

- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
  - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
  - LPEG for C parsing (see `scripts/cdoc_parser.lua`)
  - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
  - Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
   - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).
This commit is contained in:
Lewis Russell
2024-02-15 17:16:04 +00:00
committed by Lewis Russell
parent 7ad2e3c645
commit 9beb40a4db
71 changed files with 3701 additions and 3849 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
scripts/cdoc_grammar.lua Normal file
View File

@@ -0,0 +1,87 @@
--[[!
LPEG grammar for C doc comments
]]
--- @class nvim.cdoc.Param
--- @field kind 'param'
--- @field name string
--- @field desc? string
--- @class nvim.cdoc.Return
--- @field kind 'return'
--- @field desc string
--- @class nvim.cdoc.Note
--- @field desc? string
--- @alias nvim.cdoc.grammar.result
--- | nvim.cdoc.Param
--- | nvim.cdoc.Return
--- | nvim.cdoc.Note
--- @class nvim.cdoc.grammar
--- @field match fun(self, input: string): nvim.cdoc.grammar.result?
local lpeg = vim.lpeg
local P, R, S = lpeg.P, lpeg.R, lpeg.S
local Ct, Cg = lpeg.Ct, lpeg.Cg
--- @param x vim.lpeg.Pattern
local function rep(x)
return x ^ 0
end
--- @param x vim.lpeg.Pattern
local function rep1(x)
return x ^ 1
end
--- @param x vim.lpeg.Pattern
local function opt(x)
return x ^ -1
end
local nl = P('\r\n') + P('\n')
local ws = rep1(S(' \t') + nl)
local any = P(1) -- (consume one character)
local letter = R('az', 'AZ') + S('_$')
local ident = letter * rep(letter + R('09'))
local io = P('[') * (P('in') + P('out') + P('inout')) * P(']')
--- @param x string
local function Pf(x)
return opt(ws) * P(x) * opt(ws)
end
--- @type table<string,vim.lpeg.Pattern>
local v = setmetatable({}, {
__index = function(_, k)
return lpeg.V(k)
end,
})
local grammar = P {
rep1(P('@') * v.ats),
ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc,
at_param = Ct(
Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc'))
),
at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))),
at_deprecated = Ct(Cg(P('deprecated'), 'kind')),
at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')),
at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')),
at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')),
at_nodoc = Ct(Cg(P('nodoc'), 'kind')),
}
return grammar --[[@as nvim.cdoc.grammar]]

223
scripts/cdoc_parser.lua Normal file
View File

@@ -0,0 +1,223 @@
local cdoc_grammar = require('scripts.cdoc_grammar')
local c_grammar = require('src.nvim.generators.c_grammar')
--- @class nvim.cdoc.parser.param
--- @field name string
--- @field type string
--- @field desc string
--- @class nvim.cdoc.parser.return
--- @field name string
--- @field type string
--- @field desc string
--- @class nvim.cdoc.parser.note
--- @field desc string
--- @class nvim.cdoc.parser.brief
--- @field kind 'brief'
--- @field desc string
--- @class nvim.cdoc.parser.fun
--- @field name string
--- @field params nvim.cdoc.parser.param[]
--- @field returns nvim.cdoc.parser.return[]
--- @field desc string
--- @field deprecated? true
--- @field since? string
--- @field attrs? string[]
--- @field nodoc? true
--- @field notes? nvim.cdoc.parser.note[]
--- @field see? nvim.cdoc.parser.note[]
--- @class nvim.cdoc.parser.State
--- @field doc_lines? string[]
--- @field cur_obj? nvim.cdoc.parser.obj
--- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note
--- @field last_doc_item_indent? integer
--- @alias nvim.cdoc.parser.obj
--- | nvim.cdoc.parser.fun
--- | nvim.cdoc.parser.brief
--- If we collected any `---` lines. Add them to the existing (or new) object
--- Used for function/class descriptions and multiline param descriptions.
--- @param state nvim.cdoc.parser.State
local function add_doc_lines_to_obj(state)
if state.doc_lines then
state.cur_obj = state.cur_obj or {}
local cur_obj = assert(state.cur_obj)
local txt = table.concat(state.doc_lines, '\n')
if cur_obj.desc then
cur_obj.desc = cur_obj.desc .. '\n' .. txt
else
cur_obj.desc = txt
end
state.doc_lines = nil
end
end
--- @param line string
--- @param state nvim.cdoc.parser.State
local function process_doc_line(line, state)
line = line:gsub('^%s+@', '@')
local parsed = cdoc_grammar:match(line)
if not parsed then
if line:match('^ ') then
line = line:sub(2)
end
if state.last_doc_item then
if not state.last_doc_item_indent then
state.last_doc_item_indent = #line:match('^%s*') + 1
end
state.last_doc_item.desc = (state.last_doc_item.desc or '')
.. '\n'
.. line:sub(state.last_doc_item_indent or 1)
else
state.doc_lines = state.doc_lines or {}
table.insert(state.doc_lines, line)
end
return
end
state.last_doc_item_indent = nil
state.last_doc_item = nil
local kind = parsed.kind
state.cur_obj = state.cur_obj or {}
local cur_obj = assert(state.cur_obj)
if kind == 'brief' then
state.cur_obj = {
kind = 'brief',
desc = parsed.desc,
}
elseif kind == 'param' then
state.last_doc_item_indent = nil
cur_obj.params = cur_obj.params or {}
state.last_doc_item = {
name = parsed.name,
desc = parsed.desc,
}
table.insert(cur_obj.params, state.last_doc_item)
elseif kind == 'return' then
cur_obj.returns = { {
desc = parsed.desc,
} }
state.last_doc_item_indent = nil
state.last_doc_item = cur_obj.returns[1]
elseif kind == 'deprecated' then
cur_obj.deprecated = true
elseif kind == 'nodoc' then
cur_obj.nodoc = true
elseif kind == 'since' then
cur_obj.since = parsed.desc
elseif kind == 'see' then
cur_obj.see = cur_obj.see or {}
table.insert(cur_obj.see, { desc = parsed.desc })
elseif kind == 'note' then
state.last_doc_item_indent = nil
state.last_doc_item = {
desc = parsed.desc,
}
cur_obj.notes = cur_obj.notes or {}
table.insert(cur_obj.notes, state.last_doc_item)
else
error('Unhandled' .. vim.inspect(parsed))
end
end
--- @param item table
--- @param state nvim.cdoc.parser.State
local function process_proto(item, state)
state.cur_obj = state.cur_obj or {}
local cur_obj = assert(state.cur_obj)
cur_obj.name = item.name
cur_obj.params = cur_obj.params or {}
for _, p in ipairs(item.parameters) do
local param = { name = p[2], type = p[1] }
local added = false
for _, cp in ipairs(cur_obj.params) do
if cp.name == param.name then
cp.type = param.type
added = true
break
end
end
if not added then
table.insert(cur_obj.params, param)
end
end
cur_obj.returns = cur_obj.returns or { {} }
cur_obj.returns[1].type = item.return_type
for _, a in ipairs({
'fast',
'remote_only',
'lua_only',
'textlock',
'textlock_allow_cmdwin',
}) do
if item[a] then
cur_obj.attrs = cur_obj.attrs or {}
table.insert(cur_obj.attrs, a)
end
end
cur_obj.deprecated_since = item.deprecated_since
-- Remove some arguments
for i = #cur_obj.params, 1, -1 do
local p = cur_obj.params[i]
if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then
table.remove(cur_obj.params, i)
end
end
end
local M = {}
--- @param filename string
--- @return {} classes
--- @return nvim.cdoc.parser.fun[] funs
--- @return string[] briefs
function M.parse(filename)
local funs = {} --- @type nvim.cdoc.parser.fun[]
local briefs = {} --- @type string[]
local state = {} --- @type nvim.cdoc.parser.State
local txt = assert(io.open(filename, 'r')):read('*all')
local parsed = c_grammar.grammar:match(txt)
for _, item in ipairs(parsed) do
if item.comment then
process_doc_line(item.comment, state)
else
add_doc_lines_to_obj(state)
if item[1] == 'proto' then
process_proto(item, state)
table.insert(funs, state.cur_obj)
end
local cur_obj = state.cur_obj
if cur_obj and not item.static then
if cur_obj.kind == 'brief' then
table.insert(briefs, cur_obj.desc)
end
end
state = {}
end
end
return {}, funs, briefs
end
-- M.parse('src/nvim/api/vim.c')
return M

106
scripts/gen_eval_files.lua
View File

@@ -3,7 +3,6 @@
-- Generator for various vimdoc and Lua type files
local DEP_API_METADATA = 'build/api_metadata.mpack'
local DEP_API_DOC = 'runtime/doc/api.mpack'
--- @class vim.api.metadata
--- @field name string
@@ -210,44 +209,65 @@ end
--- @return table<string, vim.EvalFn>
local function get_api_meta()
local mpack_f = assert(io.open(DEP_API_METADATA, 'rb'))
local metadata = vim.mpack.decode(mpack_f:read('*all')) --[[@as vim.api.metadata[] ]]
local ret = {} --- @type table<string, vim.EvalFn>
local doc_mpack_f = assert(io.open(DEP_API_DOC, 'rb'))
local doc_metadata = vim.mpack.decode(doc_mpack_f:read('*all')) --[[@as table<string,vim.gen_vim_doc_fun>]]
local cdoc_parser = require('scripts.cdoc_parser')
for _, fun in ipairs(metadata) do
if fun.lua then
local fdoc = doc_metadata[fun.name]
local f = 'src/nvim/api'
local params = {} --- @type {[1]:string,[2]:string}[]
for _, p in ipairs(fun.parameters) do
local ptype, pname = p[1], p[2]
params[#params + 1] = {
pname,
api_type(ptype),
fdoc and fdoc.parameters_doc[pname] or nil,
}
end
local r = {
signature = 'NA',
name = fun.name,
params = params,
returns = api_type(fun.return_type),
deprecated = fun.deprecated_since ~= nil,
}
if fdoc then
if #fdoc.doc > 0 then
r.desc = table.concat(fdoc.doc, '\n')
end
r.return_desc = (fdoc['return'] or {})[1]
end
ret[fun.name] = r
local function include(fun)
if not vim.startswith(fun.name, 'nvim_') then
return false
end
if vim.tbl_contains(fun.attrs or {}, 'lua_only') then
return true
end
if vim.tbl_contains(fun.attrs or {}, 'remote_only') then
return false
end
return true
end
--- @type table<string,nvim.cdoc.parser.fun>
local functions = {}
for path, ty in vim.fs.dir(f) do
if ty == 'file' then
local filename = vim.fs.joinpath(f, path)
local _, funs = cdoc_parser.parse(filename)
for _, fn in ipairs(funs) do
if include(fn) then
functions[fn.name] = fn
end
end
end
end
for _, fun in pairs(functions) do
local deprecated = fun.deprecated_since ~= nil
local params = {} --- @type {[1]:string,[2]:string}[]
for _, p in ipairs(fun.params) do
params[#params + 1] = {
p.name,
api_type(p.type),
not deprecated and p.desc or nil,
}
end
local r = {
signature = 'NA',
name = fun.name,
params = params,
returns = api_type(fun.returns[1].type),
deprecated = deprecated,
}
if not deprecated then
r.desc = fun.desc
r.return_desc = fun.returns[1].desc
end
ret[fun.name] = r
end
return ret
end
@@ -275,12 +295,10 @@ end
--- @param fun vim.EvalFn
--- @param write fun(line: string)
local function render_api_meta(_f, fun, write)
if not vim.startswith(fun.name, 'nvim_') then
return
end
write('')
local text_utils = require('scripts.text_utils')
if vim.startswith(fun.name, 'nvim__') then
write('--- @private')
end
@@ -291,10 +309,10 @@ local function render_api_meta(_f, fun, write)
local desc = fun.desc
if desc then
desc = text_utils.md_to_vimdoc(desc, 0, 0, 74)
for _, l in ipairs(split(norm_text(desc))) do
write('--- ' .. l)
end
write('---')
end
local param_names = {} --- @type string[]
@@ -303,8 +321,11 @@ local function render_api_meta(_f, fun, write)
param_names[#param_names + 1] = p[1]
local pdesc = p[3]
if pdesc then
local pdesc_a = split(norm_text(pdesc))
write('--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' .. pdesc_a[1])
local s = '--- @param ' .. p[1] .. ' ' .. p[2] .. ' '
local indent = #('@param ' .. p[1] .. ' ')
pdesc = text_utils.md_to_vimdoc(pdesc, #s, indent, 74, true)
local pdesc_a = split(vim.trim(norm_text(pdesc)))
write(s .. pdesc_a[1])
for i = 2, #pdesc_a do
if not pdesc_a[i] then
break
@@ -317,6 +338,7 @@ local function render_api_meta(_f, fun, write)
end
if fun.returns ~= '' then
local ret_desc = fun.returns_desc and ' : ' .. fun.returns_desc or ''
ret_desc = text_utils.md_to_vimdoc(ret_desc, 0, 0, 74)
local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns
write('--- @return ' .. ret .. ret_desc)
end
@@ -328,8 +350,6 @@ end
--- @return table<string, vim.EvalFn>
local function get_api_keysets_meta()
local mpack_f = assert(io.open(DEP_API_METADATA, 'rb'))
--- @diagnostic disable-next-line:no-unknown
local metadata = assert(vim.mpack.decode(mpack_f:read('*all')))
local ret = {} --- @type table<string, vim.EvalFn>

787
scripts/gen_vimdoc.lua Executable file
View File

@@ -0,0 +1,787 @@
#!/usr/bin/env -S nvim -l
--- Generates Nvim :help docs from Lua/C docstrings
---
--- The generated :help text for each function is formatted as follows:
--- - Max width of 78 columns (`TEXT_WIDTH`).
--- - Indent with spaces (not tabs).
--- - Indent of 4 columns for body text (`INDENTATION`).
--- - Function signature and helptag (right-aligned) on the same line.
--- - Signature and helptag must have a minimum of 8 spaces between them.
--- - If the signature is too long, it is placed on the line after the helptag.
--- Signature wraps with subsequent lines indented to the open parenthesis.
--- - Subsection bodies are indented an additional 4 spaces.
--- - Body consists of function description, parameters, return description, and
--- C declaration (`INCLUDE_C_DECL`).
--- - Parameters are omitted for the `void` and `Error *` types, or if the
--- parameter is marked as [out].
--- - Each function documentation is separated by a single line.
local luacats_parser = require('scripts.luacats_parser')
local cdoc_parser = require('scripts.cdoc_parser')
local text_utils = require('scripts.text_utils')
local fmt = string.format
local wrap = text_utils.wrap
local md_to_vimdoc = text_utils.md_to_vimdoc
local TEXT_WIDTH = 78
local INDENTATION = 4
--- @class (exact) nvim.gen_vimdoc.Config
---
--- Generated documentation target, e.g. api.txt
--- @field filename string
---
--- @field section_order string[]
---
--- List of files/directories for doxygen to read, relative to `base_dir`.
--- @field files string[]
---
--- @field exclude_types? true
---
--- Section name overrides. Key: filename (e.g., vim.c)
--- @field section_name? table<string,string>
---
--- @field fn_name_pat? string
---
--- @field fn_xform? fun(fun: nvim.luacats.parser.fun)
---
--- For generated section names.
--- @field section_fmt fun(name: string): string
---
--- @field helptag_fmt fun(name: string): string
---
--- Per-function helptag.
--- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string
---
--- @field append_only? string[]
local function contains(t, xs)
return vim.tbl_contains(xs, t)
end
--- @type {level:integer, prerelease:boolean}?
local nvim_api_info_
--- @return {level: integer, prerelease:boolean}
local function nvim_api_info()
if not nvim_api_info_ then
--- @type integer?, boolean?
local level, prerelease
for l in io.lines('CMakeLists.txt') do
--- @cast l string
if level and prerelease then
break
end
local m1 = l:match('^set%(NVIM_API_LEVEL%s+(%d+)%)')
if m1 then
level = tonumber(m1) --[[@as integer]]
end
local m2 = l:match('^set%(NVIM_API_PRERELEASE%s+(%w+)%)')
if m2 then
prerelease = m2 == 'true'
end
end
nvim_api_info_ = { level = level, prerelease = prerelease }
end
return nvim_api_info_
end
--- @param fun nvim.luacats.parser.fun
--- @return string
local function fn_helptag_fmt_common(fun)
local fn_sfx = fun.table and '' or '()'
if fun.classvar then
return fmt('*%s:%s%s*', fun.classvar, fun.name, fn_sfx)
end
if fun.module then
return fmt('*%s.%s%s*', fun.module, fun.name, fn_sfx)
end
return fmt('*%s%s*', fun.name, fn_sfx)
end
--- @type table<string,nvim.gen_vimdoc.Config>
local config = {
api = {
filename = 'api.txt',
section_order = {
'vim.c',
'vimscript.c',
'command.c',
'options.c',
'buffer.c',
'extmark.c',
'window.c',
'win_config.c',
'tabpage.c',
'autocmd.c',
'ui.c',
},
exclude_types = true,
fn_name_pat = 'nvim_.*',
files = { 'src/nvim/api' },
section_name = {
['vim.c'] = 'Global',
},
section_fmt = function(name)
return name .. ' Functions'
end,
helptag_fmt = function(name)
return fmt('*api-%s*', name:lower())
end,
},
lua = {
filename = 'lua.txt',
section_order = {
'highlight.lua',
'diff.lua',
'mpack.lua',
'json.lua',
'base64.lua',
'spell.lua',
'builtin.lua',
'_options.lua',
'_editor.lua',
'_inspector.lua',
'shared.lua',
'loader.lua',
'uri.lua',
'ui.lua',
'filetype.lua',
'keymap.lua',
'fs.lua',
'glob.lua',
'lpeg.lua',
're.lua',
'regex.lua',
'secure.lua',
'version.lua',
'iter.lua',
'snippet.lua',
'text.lua',
},
files = {
'runtime/lua/vim/iter.lua',
'runtime/lua/vim/_editor.lua',
'runtime/lua/vim/_options.lua',
'runtime/lua/vim/shared.lua',
'runtime/lua/vim/loader.lua',
'runtime/lua/vim/uri.lua',
'runtime/lua/vim/ui.lua',
'runtime/lua/vim/filetype.lua',
'runtime/lua/vim/keymap.lua',
'runtime/lua/vim/fs.lua',
'runtime/lua/vim/highlight.lua',
'runtime/lua/vim/secure.lua',
'runtime/lua/vim/version.lua',
'runtime/lua/vim/_inspector.lua',
'runtime/lua/vim/snippet.lua',
'runtime/lua/vim/text.lua',
'runtime/lua/vim/glob.lua',
'runtime/lua/vim/_meta/builtin.lua',
'runtime/lua/vim/_meta/diff.lua',
'runtime/lua/vim/_meta/mpack.lua',
'runtime/lua/vim/_meta/json.lua',
'runtime/lua/vim/_meta/base64.lua',
'runtime/lua/vim/_meta/regex.lua',
'runtime/lua/vim/_meta/lpeg.lua',
'runtime/lua/vim/_meta/re.lua',
'runtime/lua/vim/_meta/spell.lua',
},
fn_xform = function(fun)
if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then
fun.module = 'vim'
end
if fun.module == 'vim' and contains(fun.name, { 'cmd', 'inspect' }) then
fun.table = nil
end
if fun.classvar or vim.startswith(fun.name, 'vim.') or fun.module == 'vim.iter' then
return
end
fun.name = fmt('%s.%s', fun.module, fun.name)
end,
section_name = {
['_inspector.lua'] = 'inspector',
},
section_fmt = function(name)
name = name:lower()
if name == '_editor' then
return 'Lua module: vim'
elseif name == '_options' then
return 'LUA-VIMSCRIPT BRIDGE'
elseif name == 'builtin' then
return 'VIM'
end
if
contains(name, {
'highlight',
'mpack',
'json',
'base64',
'diff',
'spell',
'regex',
'lpeg',
're',
})
then
return 'VIM.' .. name:upper()
end
return 'Lua module: vim.' .. name
end,
helptag_fmt = function(name)
if name == '_editor' then
return '*lua-vim*'
elseif name == '_options' then
return '*lua-vimscript*'
end
return '*vim.' .. name:lower() .. '*'
end,
fn_helptag_fmt = function(fun)
local name = fun.name
if vim.startswith(name, 'vim.') then
local fn_sfx = fun.table and '' or '()'
return fmt('*%s%s*', name, fn_sfx)
elseif fun.classvar == 'Option' then
return fmt('*vim.opt:%s()*', name)
end
return fn_helptag_fmt_common(fun)
end,
append_only = {
'shared.lua',
},
},
lsp = {
filename = 'lsp.txt',
section_order = {
'lsp.lua',
'buf.lua',
'diagnostic.lua',
'codelens.lua',
'inlay_hint.lua',
'tagfunc.lua',
'semantic_tokens.lua',
'handlers.lua',
'util.lua',
'log.lua',
'rpc.lua',
'protocol.lua',
},
files = {
'runtime/lua/vim/lsp',
'runtime/lua/vim/lsp.lua',
},
fn_xform = function(fun)
fun.name = fun.name:gsub('result%.', '')
end,
section_fmt = function(name)
if name:lower() == 'lsp' then
return 'Lua module: vim.lsp'
end
return 'Lua module: vim.lsp.' .. name:lower()
end,
helptag_fmt = function(name)
if name:lower() == 'lsp' then
return '*lsp-core*'
end
return fmt('*lsp-%s*', name:lower())
end,
},
diagnostic = {
filename = 'diagnostic.txt',
section_order = {
'diagnostic.lua',
},
files = { 'runtime/lua/vim/diagnostic.lua' },
section_fmt = function()
return 'Lua module: vim.diagnostic'
end,
helptag_fmt = function()
return '*diagnostic-api*'
end,
},
treesitter = {
filename = 'treesitter.txt',
section_order = {
'treesitter.lua',
'language.lua',
'query.lua',
'highlighter.lua',
'languagetree.lua',
'dev.lua',
},
files = {
'runtime/lua/vim/treesitter.lua',
'runtime/lua/vim/treesitter/',
},
section_fmt = function(name)
if name:lower() == 'treesitter' then
return 'Lua module: vim.treesitter'
end
return 'Lua module: vim.treesitter.' .. name:lower()
end,
helptag_fmt = function(name)
if name:lower() == 'treesitter' then
return '*lua-treesitter-core*'
end
return '*lua-treesitter-' .. name:lower() .. '*'
end,
},
}
--- @param ty string
--- @param generics table<string,string>
--- @return string
local function replace_generics(ty, generics)
if ty:sub(-2) == '[]' then
local ty0 = ty:sub(1, -3)
if generics[ty0] then
return generics[ty0] .. '[]'
end
elseif ty:sub(-1) == '?' then
local ty0 = ty:sub(1, -2)
if generics[ty0] then
return generics[ty0] .. '?'
end
end
return generics[ty] or ty
end
--- @param ty string
--- @param generics? table<string,string>
local function render_type(ty, generics)
if generics then
ty = replace_generics(ty, generics)
end
ty = ty:gsub('%s*|%s*nil', '?')
ty = ty:gsub('nil%s*|%s*(.*)', '%1?')
ty = ty:gsub('%s*|%s*', '|')
return fmt('(`%s`)', ty)
end
--- @param p nvim.luacats.parser.param|nvim.luacats.parser.field
local function should_render_param(p)
return not p.access and not contains(p.name, { '_', 'self' })
end
--- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[]
--- @param generics? table<string,string>
--- @param exclude_types? true
local function render_fields_or_params(xs, generics, exclude_types)
local ret = {} --- @type string[]
xs = vim.tbl_filter(should_render_param, xs)
local indent = 0
for _, p in ipairs(xs) do
if p.type or p.desc then
indent = math.max(indent, #p.name + 3)
end
if exclude_types then
p.type = nil
end
end
for _, p in ipairs(xs) do
local nm, ty = p.name, p.type
local desc = p.desc
local pnm = fmt(' • %-' .. indent .. 's', '{' .. nm .. '}')
if ty then
local pty = render_type(ty, generics)
if desc then
desc = fmt('%s %s', pty, desc)
table.insert(ret, pnm)
table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true))
else
table.insert(ret, fmt('%s %s\n', pnm, pty))
end
else
if desc then
table.insert(ret, pnm)
table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true))
end
end
end
return table.concat(ret)
end
-- --- @param class lua2vimdoc.class
-- local function render_class(class)
-- writeln(fmt('*%s*', class.name))
-- writeln()
-- if #class.fields > 0 then
-- writeln(' Fields: ~')
-- render_fields_or_params(class.fields)
-- end
-- writeln()
-- end
-- --- @param cls table<string,lua2vimdoc.class>
-- local function render_classes(cls)
-- --- @diagnostic disable-next-line:no-unknown
-- for _, class in vim.spairs(cls) do
-- render_class(class)
-- end
-- end
--- @param fun nvim.luacats.parser.fun
--- @param cfg nvim.gen_vimdoc.Config
local function render_fun_header(fun, cfg)
local ret = {} --- @type string[]
local args = {} --- @type string[]
for _, p in ipairs(fun.params or {}) do
if p.name ~= 'self' then
args[#args + 1] = fmt('{%s}', p.name:gsub('%?$', ''))
end
end
local nm = fun.name
if fun.classvar then
nm = fmt('%s:%s', fun.classvar, nm)
end
local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')'
if not cfg.fn_helptag_fmt then
cfg.fn_helptag_fmt = fn_helptag_fmt_common
end
local tag = cfg.fn_helptag_fmt(fun)
if #proto + #tag > TEXT_WIDTH - 8 then
table.insert(ret, fmt('%78s\n', tag))
local name, pargs = proto:match('([^(]+%()(.*)')
table.insert(ret, name)
table.insert(ret, wrap(pargs, 0, #name, TEXT_WIDTH))
else
local pad = TEXT_WIDTH - #proto - #tag
table.insert(ret, proto .. string.rep(' ', pad) .. tag)
end
return table.concat(ret)
end
--- @param returns nvim.luacats.parser.return[]
--- @param generics? table<string,string>
--- @param exclude_types boolean
local function render_returns(returns, generics, exclude_types)
local ret = {} --- @type string[]
returns = vim.deepcopy(returns)
if exclude_types then
for _, r in ipairs(returns) do
r.type = nil
end
end
if #returns > 1 then
table.insert(ret, ' Return (multiple): ~\n')
elseif #returns == 1 and next(returns[1]) then
table.insert(ret, ' Return: ~\n')
end
for _, p in ipairs(returns) do
local rnm, ty, desc = p.name, p.type, p.desc
local blk = ''
if ty then
blk = render_type(ty, generics)
end
if rnm then
blk = blk .. ' ' .. rnm
end
if desc then
blk = blk .. ' ' .. desc
end
table.insert(ret, md_to_vimdoc(blk, 8, 8, TEXT_WIDTH, true))
end
return table.concat(ret)
end
--- @param fun nvim.luacats.parser.fun
--- @param cfg nvim.gen_vimdoc.Config
local function render_fun(fun, cfg)
if fun.access or fun.deprecated or fun.nodoc then
return
end
if cfg.fn_name_pat and not fun.name:match(cfg.fn_name_pat) then
return
end
if vim.startswith(fun.name, '_') or fun.name:find('[:.]_') then
return
end
local ret = {} --- @type string[]
table.insert(ret, render_fun_header(fun, cfg))
table.insert(ret, '\n')
if fun.desc then
table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH))
end
if fun.since then
local since = tonumber(fun.since)
local info = nvim_api_info()
if since and (since > info.level or since == info.level and info.prerelease) then
fun.notes = fun.notes or {}
table.insert(fun.notes, { desc = 'This API is pre-release (unstable).' })
end
end
if fun.notes then
table.insert(ret, '\n Note: ~\n')
for _, p in ipairs(fun.notes) do
table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true))
end
end
if fun.attrs then
table.insert(ret, '\n Attributes: ~\n')
for _, attr in ipairs(fun.attrs) do
local attr_str = ({
textlock = 'not allowed when |textlock| is active or in the |cmdwin|',
textlock_allow_cmdwin = 'not allowed when |textlock| is active',
fast = '|api-fast|',
remote_only = '|RPC| only',
lua_only = 'Lua |vim.api| only',
})[attr] or attr
table.insert(ret, fmt(' %s\n', attr_str))
end
end
if fun.params and #fun.params > 0 then
local param_txt = render_fields_or_params(fun.params, fun.generics, cfg.exclude_types)
if not param_txt:match('^%s*$') then
table.insert(ret, '\n Parameters: ~\n')
ret[#ret + 1] = param_txt
end
end
if fun.returns then
local txt = render_returns(fun.returns, fun.generics, cfg.exclude_types)
if not txt:match('^%s*$') then
table.insert(ret, '\n')
ret[#ret + 1] = txt
end
end
if fun.see then
table.insert(ret, '\n See also: ~\n')
for _, p in ipairs(fun.see) do
table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true))
end
end
table.insert(ret, '\n')
return table.concat(ret)
end
--- @param funs nvim.luacats.parser.fun[]
--- @param cfg nvim.gen_vimdoc.Config
local function render_funs(funs, cfg)
local ret = {} --- @type string[]
for _, f in ipairs(funs) do
if cfg.fn_xform then
cfg.fn_xform(f)
end
ret[#ret + 1] = render_fun(f, cfg)
end
-- Sort via prototype
table.sort(ret, function(a, b)
local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n')
local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n')
return a1:lower() < b1:lower()
end)
return table.concat(ret)
end
--- @return string
local function get_script_path()
local str = debug.getinfo(2, 'S').source:sub(2)
return str:match('(.*[/\\])') or './'
end
local script_path = get_script_path()
local base_dir = vim.fs.dirname(assert(vim.fs.dirname(script_path)))
local function delete_lines_below(doc_file, tokenstr)
local lines = {} --- @type string[]
local found = false
for line in io.lines(doc_file) do
if line:find(vim.pesc(tokenstr)) then
found = true
break
end
lines[#lines + 1] = line
end
if not found then
error(fmt('not found: %s in %s', tokenstr, doc_file))
end
lines[#lines] = nil
local fp = assert(io.open(doc_file, 'w'))
fp:write(table.concat(lines, '\n'))
fp:write('\n')
fp:close()
end
--- @param x string
local function mktitle(x)
if x == 'ui' then
return 'UI'
end
return x:sub(1, 1):upper() .. x:sub(2)
end
--- @class nvim.gen_vimdoc.Section
--- @field name string
--- @field title string
--- @field help_tag string
--- @field funs_txt string
--- @field doc? string[]
--- @param filename string
--- @param cfg nvim.gen_vimdoc.Config
--- @param section_docs table<string,nvim.gen_vimdoc.Section>
--- @param funs_txt string
--- @return nvim.gen_vimdoc.Section?
local function make_section(filename, cfg, section_docs, funs_txt)
-- filename: e.g., 'autocmd.c'
-- name: e.g. 'autocmd'
local name = filename:match('(.*)%.[a-z]+')
-- Formatted (this is what's going to be written in the vimdoc)
-- e.g., "Autocmd Functions"
local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name)
-- section tag: e.g., "*api-autocmd*"
local help_tag = cfg.helptag_fmt(sectname)
if funs_txt == '' and #section_docs == 0 then
return
end
return {
name = sectname,
title = cfg.section_fmt(sectname),
help_tag = help_tag,
funs_txt = funs_txt,
doc = section_docs,
}
end
--- @param section nvim.gen_vimdoc.Section
--- @param add_header? boolean
local function render_section(section, add_header)
local doc = {} --- @type string[]
if add_header ~= false then
vim.list_extend(doc, {
string.rep('=', TEXT_WIDTH),
'\n',
section.title,
fmt('%' .. (TEXT_WIDTH - section.title:len()) .. 's', section.help_tag),
})
end
if section.doc and #section.doc > 0 then
table.insert(doc, '\n\n')
vim.list_extend(doc, section.doc)
end
if section.funs_txt then
table.insert(doc, '\n\n')
table.insert(doc, section.funs_txt)
end
return table.concat(doc)
end
local parsers = {
lua = luacats_parser.parse,
c = cdoc_parser.parse,
h = cdoc_parser.parse,
}
--- @param files string[]
local function expand_files(files)
for k, f in pairs(files) do
if vim.fn.isdirectory(f) == 1 then
table.remove(files, k)
for path, ty in vim.fs.dir(f) do
if ty == 'file' then
table.insert(files, vim.fs.joinpath(f, path))
end
end
end
end
end
--- @param cfg nvim.gen_vimdoc.Config
local function gen_target(cfg)
local sections = {} --- @type table<string,nvim.gen_vimdoc.Section>
expand_files(cfg.files)
for _, f in pairs(cfg.files) do
local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']]
local parser = assert(parsers[ext])
local _, funs, briefs = parser(f)
local briefs_txt = {} --- @type string[]
for _, b in ipairs(briefs) do
briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH)
end
local funs_txt = render_funs(funs, cfg)
-- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua`
local f_base = assert(vim.fs.basename(f))
sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt)
end
local first_section_tag = sections[cfg.section_order[1]].help_tag
local docs = {} --- @type string[]
for _, f in ipairs(cfg.section_order) do
local section = sections[f]
if section then
local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f)
table.insert(docs, render_section(section, add_sep_and_header))
end
end
table.insert(
docs,
fmt(' vim:tw=78:ts=8:sw=%d:sts=%d:et:ft=help:norl:\n', INDENTATION, INDENTATION)
)
local doc_file = vim.fs.joinpath(base_dir, 'runtime', 'doc', cfg.filename)
if vim.uv.fs_stat(doc_file) then
delete_lines_below(doc_file, first_section_tag)
end
local fp = assert(io.open(doc_file, 'a'))
fp:write(table.concat(docs, '\n'))
fp:close()
end
local function run()
for _, cfg in pairs(config) do
gen_target(cfg)
end
end
run()

1766
scripts/gen_vimdoc.py
View File

File diff suppressed because it is too large Load Diff

544
scripts/lua2dox.lua
View File

@@ -1,544 +0,0 @@
-----------------------------------------------------------------------------
-- Copyright (C) 2012 by Simon Dales --
-- simon@purrsoft.co.uk --
-- --
-- This program is free software; you can redistribute it and/or modify --
-- it under the terms of the GNU General Public License as published by --
-- the Free Software Foundation; either version 2 of the License, or --
-- (at your option) any later version. --
-- --
-- This program is distributed in the hope that it will be useful, --
-- but WITHOUT ANY WARRANTY; without even the implied warranty of --
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --
-- GNU General Public License for more details. --
-- --
-- You should have received a copy of the GNU General Public License --
-- along with this program; if not, write to the --
-- Free Software Foundation, Inc., --
-- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. --
-----------------------------------------------------------------------------
--[[!
Lua-to-Doxygen converter
Partially from lua2dox
http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm
RUNNING
-------
This script "lua2dox.lua" gets called by "gen_vimdoc.py".
DEBUGGING/DEVELOPING
---------------------
1. To debug, run gen_vimdoc.py with --keep-tmpfiles:
python3 scripts/gen_vimdoc.py -t treesitter --keep-tmpfiles
2. The filtered result will be written to ./tmp-lua2dox-doc/….lua.c
Doxygen must be on your system. You can experiment like so:
- Run "doxygen -g" to create a default Doxyfile.
- Then alter it to let it recognise lua. Add the following line:
FILE_PATTERNS = *.lua
- Then run "doxygen".
The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language.
It only has to be good enough for doxygen to see it as legal.
One limitation is that each line is treated separately (except for long comments).
The implication is that class and function declarations must be on the same line.
There is hack that will insert the "missing" close paren.
The effect is that you will get the function documented, but not with the parameter list you might expect.
]]
local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' }
local luacats_parser = require('src/nvim/generators/luacats_grammar')
local debug_outfile = nil --- @type string?
local debug_output = {}
--- write to stdout
--- @param str? string
local function write(str)
if not str then
return
end
io.write(str)
if debug_outfile then
table.insert(debug_output, str)
end
end
--- write to stdout
--- @param str? string
local function writeln(str)
write(str)
write('\n')
end
--- an input file buffer
--- @class StreamRead
--- @field currentLine string?
--- @field contentsLen integer
--- @field currentLineNo integer
--- @field filecontents string[]
local StreamRead = {}
--- @return StreamRead
--- @param filename string
function StreamRead.new(filename)
assert(filename, ('invalid file: %s'):format(filename))
-- get lines from file
-- syphon lines to our table
local filecontents = {} --- @type string[]
for line in io.lines(filename) do
filecontents[#filecontents + 1] = line
end
return setmetatable({
filecontents = filecontents,
contentsLen = #filecontents,
currentLineNo = 1,
}, { __index = StreamRead })
end
-- get a line
function StreamRead:getLine()
if self.currentLine then
self.currentLine = nil
return self.currentLine
end
-- get line
if self.currentLineNo <= self.contentsLen then
local line = self.filecontents[self.currentLineNo]
self.currentLineNo = self.currentLineNo + 1
return line
end
return ''
end
-- save line fragment
--- @param line_fragment string
function StreamRead:ungetLine(line_fragment)
self.currentLine = line_fragment
end
-- is it eof?
function StreamRead:eof()
return not self.currentLine and self.currentLineNo > self.contentsLen
end
-- input filter
--- @class Lua2DoxFilter
local Lua2DoxFilter = {
generics = {}, --- @type table<string,string>
block_ignore = false, --- @type boolean
}
setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter })
function Lua2DoxFilter:reset()
self.generics = {}
self.block_ignore = false
end
--- trim comment off end of string
---
--- @param line string
--- @return string, string?
local function removeCommentFromLine(line)
local pos_comment = line:find('%-%-')
if not pos_comment then
return line
end
return line:sub(1, pos_comment - 1), line:sub(pos_comment)
end
--- @param parsed luacats.Return
--- @return string
local function get_return_type(parsed)
local elems = {} --- @type string[]
for _, v in ipairs(parsed) do
local e = v.type --- @type string
if v.name then
e = e .. ' ' .. v.name --- @type string
end
elems[#elems + 1] = e
end
return '(' .. table.concat(elems, ', ') .. ')'
end
--- @param name string
--- @return string
local function process_name(name, optional)
if optional then
name = name:sub(1, -2) --- @type string
end
return name
end
--- @param ty string
--- @param generics table<string,string>
--- @return string
local function process_type(ty, generics, optional)
-- replace generic types
for k, v in pairs(generics) do
ty = ty:gsub(k, v) --- @type string
end
-- strip parens
ty = ty:gsub('^%((.*)%)$', '%1')
if optional and not ty:find('nil') then
ty = ty .. '?'
end
-- remove whitespace in unions
ty = ty:gsub('%s*|%s*', '|')
-- replace '|nil' with '?'
ty = ty:gsub('|nil', '?')
ty = ty:gsub('nil|(.*)', '%1?')
return '(`' .. ty .. '`)'
end
--- @param parsed luacats.Param
--- @param generics table<string,string>
--- @return string
local function process_param(parsed, generics)
local name, ty = parsed.name, parsed.type
local optional = vim.endswith(name, '?')
return table.concat({
'/// @param',
process_name(name, optional),
process_type(ty, generics, optional),
parsed.desc,
}, ' ')
end
--- @param parsed luacats.Return
--- @param generics table<string,string>
--- @return string
local function process_return(parsed, generics)
local ty, name --- @type string, string
if #parsed == 1 then
ty, name = parsed[1].type, parsed[1].name or ''
else
ty, name = get_return_type(parsed), ''
end
local optional = vim.endswith(name, '?')
return table.concat({
'/// @return',
process_type(ty, generics, optional),
process_name(name, optional),
parsed.desc,
}, ' ')
end
--- Processes "@…" directives in a docstring line.
---
--- @param line string
--- @return string?
function Lua2DoxFilter:process_magic(line)
line = line:gsub('^%s+@', '@')
line = line:gsub('@package', '@private')
line = line:gsub('@nodoc', '@private')
if self.block_ignore then
return '// gg:" ' .. line .. '"'
end
if not vim.startswith(line, '@') then -- it's a magic comment
return '/// ' .. line
end
local magic_split = vim.split(line, ' ', { plain = true })
local directive = magic_split[1]
if
vim.list_contains({
'@cast',
'@diagnostic',
'@overload',
'@meta',
'@type',
}, directive)
then
-- Ignore LSP directives
return '// gg:"' .. line .. '"'
elseif directive == '@defgroup' or directive == '@addtogroup' then
-- Can't use '.' in defgroup, so convert to '--'
return '/// ' .. line:gsub('%.', '-dot-')
end
if directive == '@alias' then
-- this contiguous block should be all ignored.
self.block_ignore = true
return '// gg:"' .. line .. '"'
end
-- preprocess line before parsing
if directive == '@param' or directive == '@return' then
for _, type in ipairs(TYPES) do
line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2')
line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2')
line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2')
line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1')
line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1')
line = line:gsub('^@return%s+.*%((' .. type .. '%?)%)', '@return %1')
end
end
local parsed = luacats_parser:match(line)
if not parsed then
return '/// ' .. line
end
local kind = parsed.kind
if kind == 'generic' then
self.generics[parsed.name] = parsed.type or 'any'
return
elseif kind == 'param' then
return process_param(parsed --[[@as luacats.Param]], self.generics)
elseif kind == 'return' then
return process_return(parsed --[[@as luacats.Return]], self.generics)
end
error(string.format('unhandled parsed line %q: %s', line, parsed))
end
--- @param line string
--- @param in_stream StreamRead
--- @return string
function Lua2DoxFilter:process_block_comment(line, in_stream)
local comment_parts = {} --- @type string[]
local done --- @type boolean?
while not done and not in_stream:eof() do
local thisComment --- @type string?
local closeSquare = line:find(']]')
if not closeSquare then -- need to look on another line
thisComment = line .. '\n'
line = in_stream:getLine()
else
thisComment = line:sub(1, closeSquare - 1)
done = true
-- unget the tail of the line
-- in most cases it's empty. This may make us less efficient but
-- easier to program
in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2)))
end
comment_parts[#comment_parts + 1] = thisComment
end
local comment = table.concat(comment_parts)
if comment:sub(1, 1) == '@' then -- it's a long magic comment
return '/*' .. comment .. '*/ '
end
-- discard
return '/* zz:' .. comment .. '*/ '
end
--- @param line string
--- @return string
function Lua2DoxFilter:process_function_header(line)
local pos_fn = assert(line:find('function'))
-- we've got a function
local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8)))
if fn:sub(1, 1) == '(' then
-- it's an anonymous function
return '// ZZ: ' .. line
end
-- fn has a name, so is interesting
-- want to fix for iffy declarations
if fn:find('[%({]') then
-- we might have a missing close paren
if not fn:find('%)') then
fn = fn .. ' ___MissingCloseParenHere___)'
end
end
-- Big hax
if fn:find(':') then
fn = fn:gsub(':', '.', 1)
local paren_start = fn:find('(', 1, true)
local paren_finish = fn:find(')', 1, true)
-- Nothing in between the parens
local comma --- @type string
if paren_finish == paren_start + 1 then
comma = ''
else
comma = ', '
end
fn = fn:sub(1, paren_start) .. 'self' .. comma .. fn:sub(paren_start + 1)
end
if line:match('local') then
-- Special: tell gen_vimdoc.py this is a local function.
return 'local_function ' .. fn .. '{}'
end
-- add vanilla function
return 'function ' .. fn .. '{}'
end
--- @param line string
--- @param in_stream StreamRead
--- @return string?
function Lua2DoxFilter:process_line(line, in_stream)
local line_raw = line
line = vim.trim(line)
if vim.startswith(line, '---') then
return Lua2DoxFilter:process_magic(line:sub(4))
end
if vim.startswith(line, '--' .. '[[') then -- it's a long comment
return Lua2DoxFilter:process_block_comment(line:sub(5), in_stream)
end
-- Hax... I'm sorry
-- M.fun = vim.memoize(function(...)
-- ->
-- function M.fun(...)
line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)')
if line:find('^function') or line:find('^local%s+function') then
return Lua2DoxFilter:process_function_header(line)
end
if not line:match('^local') then
local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=')
if v and v:match('%.') then
-- Special: this lets gen_vimdoc.py handle tables.
return 'table ' .. v .. '() {}'
end
end
if #line > 0 then -- we don't know what this line means, so just comment it out
return '// zz: ' .. line
end
return ''
end
-- Processes the file and writes filtered output to stdout.
---@param filename string
function Lua2DoxFilter:filter(filename)
local in_stream = StreamRead.new(filename)
local last_was_magic = false
while not in_stream:eof() do
local line = in_stream:getLine()
local out_line = self:process_line(line, in_stream)
if not vim.startswith(vim.trim(line), '---') then
self:reset()
end
if out_line then
-- Ensure all magic blocks associate with some object to prevent doxygen
-- from getting confused.
if vim.startswith(out_line, '///') then
last_was_magic = true
else
if last_was_magic and out_line:match('^// zz: [^-]+') then
writeln('local_function _ignore() {}')
end
last_was_magic = false
end
writeln(out_line)
end
end
end
--- @class TApp
--- @field timestamp string|osdate
--- @field name string
--- @field version string
--- @field copyright string
--- this application
local TApp = {
timestamp = os.date('%c %Z', os.time()),
name = 'Lua2DoX',
version = '0.2 20130128',
copyright = 'Copyright (c) Simon Dales 2012-13',
}
setmetatable(TApp, { __index = TApp })
function TApp:getRunStamp()
return self.name .. ' (' .. self.version .. ') ' .. self.timestamp
end
function TApp:getVersion()
return self.name .. ' (' .. self.version .. ') '
end
--main
if arg[1] == '--help' then
writeln(TApp:getVersion())
writeln(TApp.copyright)
writeln([[
run as:
nvim -l scripts/lua2dox.lua <param>
--------------
Param:
<filename> : interprets filename
--version : show version/copyright info
--help : this help text]])
elseif arg[1] == '--version' then
writeln(TApp:getVersion())
writeln(TApp.copyright)
else -- It's a filter.
local filename = arg[1]
if arg[2] == '--outdir' then
local outdir = arg[3]
if
type(outdir) ~= 'string'
or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir))
then
error(('invalid --outdir: "%s"'):format(tostring(outdir)))
end
vim.fn.mkdir(outdir, 'p')
debug_outfile = string.format('%s/%s.c', outdir, vim.fs.basename(filename))
end
Lua2DoxFilter:filter(filename)
-- output the tail
writeln('// #######################')
writeln('// app run:' .. TApp:getRunStamp())
writeln('// #######################')
writeln()
if debug_outfile then
local f = assert(io.open(debug_outfile, 'w'))
f:write(table.concat(debug_output))
f:close()
end
end

218
scripts/luacats_grammar.lua Normal file
View File

@@ -0,0 +1,218 @@
--[[!
LPEG grammar for LuaCATS
]]
local lpeg = vim.lpeg
local P, R, S = lpeg.P, lpeg.R, lpeg.S
local Ct, Cg = lpeg.Ct, lpeg.Cg
--- @param x vim.lpeg.Pattern
local function rep(x)
return x ^ 0
end
--- @param x vim.lpeg.Pattern
local function rep1(x)
return x ^ 1
end
--- @param x vim.lpeg.Pattern
local function opt(x)
return x ^ -1
end
local nl = P('\r\n') + P('\n')
local ws = rep1(S(' \t') + nl)
local fill = opt(ws)
local any = P(1) -- (consume one character)
local letter = R('az', 'AZ') + S('_$')
local num = R('09')
local ident = letter * rep(letter + num + S '-.')
local string_single = P "'" * rep(any - P "'") * P "'"
local string_double = P '"' * rep(any - P '"') * P '"'
local literal = (string_single + string_double + (opt(P '-') * num) + P 'false' + P 'true')
local lname = (ident + P '...') * opt(P '?')
--- @param x string
local function Pf(x)
return fill * P(x) * fill
end
--- @param x string
local function Sf(x)
return fill * S(x) * fill
end
--- @param x vim.lpeg.Pattern
local function comma(x)
return x * rep(Pf ',' * x)
end
--- @param x vim.lpeg.Pattern
local function parenOpt(x)
return (Pf('(') * x * fill * P(')')) + x
end
--- @type table<string,vim.lpeg.Pattern>
local v = setmetatable({}, {
__index = function(_, k)
return lpeg.V(k)
end,
})
local desc_delim = Sf '#:' + ws
--- @class nvim.luacats.Param
--- @field kind 'param'
--- @field name string
--- @field type string
--- @field desc? string
--- @class nvim.luacats.Return
--- @field kind 'return'
--- @field [integer] { type: string, name?: string}
--- @field desc? string
--- @class nvim.luacats.Generic
--- @field kind 'generic'
--- @field name string
--- @field type? string
--- @class nvim.luacats.Class
--- @field kind 'class'
--- @field name string
--- @field parent? string
--- @class nvim.luacats.Field
--- @field kind 'field'
--- @field name string
--- @field type string
--- @field desc? string
--- @field access? 'private'|'protected'|'package'
--- @class nvim.luacats.Note
--- @field desc? string
--- @alias nvim.luacats.grammar.result
--- | nvim.luacats.Param
--- | nvim.luacats.Return
--- | nvim.luacats.Generic
--- | nvim.luacats.Class
--- | nvim.luacats.Field
--- | nvim.luacats.Note
--- @class nvim.luacats.grammar
--- @field match fun(self, input: string): nvim.luacats.grammar.result?
local grammar = P {
rep1(P('@') * (v.ats + v.ext_ats)),
ats = v.at_param
+ v.at_return
+ v.at_type
+ v.at_cast
+ v.at_generic
+ v.at_class
+ v.at_field
+ v.at_access
+ v.at_deprecated
+ v.at_alias
+ v.at_enum
+ v.at_see
+ v.at_diagnostic
+ v.at_overload
+ v.at_meta,
ext_ats = v.ext_at_note + v.ext_at_since + v.ext_at_nodoc + v.ext_at_brief,
at_param = Ct(
Cg(P('param'), 'kind')
* ws
* Cg(lname, 'name')
* ws
* parenOpt(Cg(v.ltype, 'type'))
* opt(desc_delim * Cg(rep(any), 'desc'))
),
at_return = Ct(
Cg(P('return'), 'kind')
* ws
* parenOpt(comma(Ct(Cg(v.ltype, 'type') * opt(ws * Cg(ident, 'name')))))
* opt(desc_delim * Cg(rep(any), 'desc'))
),
at_type = Ct(
Cg(P('type'), 'kind')
* ws
* parenOpt(comma(Ct(Cg(v.ltype, 'type'))))
* opt(desc_delim * Cg(rep(any), 'desc'))
),
at_cast = Ct(
Cg(P('cast'), 'kind') * ws * Cg(lname, 'name') * ws * opt(Sf('+-')) * Cg(v.ltype, 'type')
),
at_generic = Ct(
Cg(P('generic'), 'kind') * ws * Cg(ident, 'name') * opt(Pf ':' * Cg(v.ltype, 'type'))
),
at_class = Ct(
Cg(P('class'), 'kind')
* ws
* opt(P('(exact)') * ws)
* Cg(lname, 'name')
* opt(Pf(':') * Cg(lname, 'parent'))
),
at_field = Ct(
Cg(P('field'), 'kind')
* ws
* opt(Cg(Pf('private') + Pf('package') + Pf('protected'), 'access'))
* Cg(lname, 'name')
* ws
* Cg(v.ltype, 'type')
* opt(desc_delim * Cg(rep(any), 'desc'))
),
at_access = Ct(Cg(P('private') + P('protected') + P('package'), 'kind')),
at_deprecated = Ct(Cg(P('deprecated'), 'kind')),
-- Types may be provided on subsequent lines
at_alias = Ct(Cg(P('alias'), 'kind') * ws * Cg(lname, 'name') * opt(ws * Cg(v.ltype, 'type'))),
at_enum = Ct(Cg(P('enum'), 'kind') * ws * Cg(lname, 'name')),
at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')),
at_diagnostic = Ct(Cg(P('diagnostic'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')),
at_overload = Ct(Cg(P('overload'), 'kind') * ws * Cg(v.ltype, 'type')),
at_meta = Ct(Cg(P('meta'), 'kind')),
--- Custom extensions
ext_at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')),
-- TODO only consume 1 line
ext_at_since = Ct(Cg(P('since'), 'kind') * ws * Cg(rep(any), 'desc')),
ext_at_nodoc = Ct(Cg(P('nodoc'), 'kind')),
ext_at_brief = Ct(Cg(P('brief'), 'kind') * opt(ws * Cg(rep(any), 'desc'))),
ltype = v.ty_union + Pf '(' * v.ty_union * fill * P ')',
ty_union = v.ty_opt * rep(Pf '|' * v.ty_opt),
ty = v.ty_fun + ident + v.ty_table + literal,
ty_param = Pf '<' * comma(v.ltype) * fill * P '>',
ty_opt = v.ty * opt(v.ty_param) * opt(P '[]') * opt(P '?'),
table_key = (Pf '[' * literal * Pf ']') + lname,
table_elem = v.table_key * Pf ':' * v.ltype,
ty_table = Pf '{' * comma(v.table_elem) * Pf '}',
fun_param = lname * opt(Pf ':' * v.ltype),
ty_fun = Pf 'fun(' * rep(comma(v.fun_param)) * fill * P ')' * opt(Pf ':' * comma(v.ltype)),
}
return grammar --[[@as nvim.luacats.grammar]]

521
scripts/luacats_parser.lua Normal file
View File

@@ -0,0 +1,521 @@
local luacats_grammar = require('scripts.luacats_grammar')
--- @class nvim.luacats.parser.param
--- @field name string
--- @field type string
--- @field desc string
--- @class nvim.luacats.parser.return
--- @field name string
--- @field type string
--- @field desc string
--- @class nvim.luacats.parser.note
--- @field desc string
--- @class nvim.luacats.parser.brief
--- @field kind 'brief'
--- @field desc string
--- @class nvim.luacats.parser.alias
--- @field kind 'alias'
--- @field type string
--- @field desc string
--- @class nvim.luacats.parser.fun
--- @field name string
--- @field params nvim.luacats.parser.param[]
--- @field returns nvim.luacats.parser.return[]
--- @field desc string
--- @field access? 'private'|'package'|'protected'
--- @field class? string
--- @field module? string
--- @field modvar? string
--- @field classvar? string
--- @field deprecated? true
--- @field since? string
--- @field attrs? string[]
--- @field nodoc? true
--- @field generics? table<string,string>
--- @field table? true
--- @field notes? nvim.luacats.parser.note[]
--- @field see? nvim.luacats.parser.note[]
--- @class nvim.luacats.parser.field
--- @field name string
--- @field type string
--- @field desc string
--- @field access? 'private'|'package'|'protected'
--- @class nvim.luacats.parser.class
--- @field kind 'class'
--- @field name string
--- @field desc string
--- @field fields nvim.luacats.parser.field[]
--- @field notes? string[]
--- @class nvim.luacats.parser.State
--- @field doc_lines? string[]
--- @field cur_obj? nvim.luacats.parser.obj
--- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note
--- @field last_doc_item_indent? integer
--- @alias nvim.luacats.parser.obj
--- | nvim.luacats.parser.class
--- | nvim.luacats.parser.fun
--- | nvim.luacats.parser.brief
-- Remove this when we document classes properly
--- Some doc lines have the form:
--- param name some.complex.type (table) description
--- if so then transform the line to remove the complex type:
--- param name (table) description
--- @param line string
local function use_type_alt(line)
for _, type in ipairs({ 'table', 'function' }) do
line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2')
line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2')
line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2')
line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1')
line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1')
line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1')
end
return line
end
--- If we collected any `---` lines. Add them to the existing (or new) object
--- Used for function/class descriptions and multiline param descriptions.
--- @param state nvim.luacats.parser.State
local function add_doc_lines_to_obj(state)
if state.doc_lines then
state.cur_obj = state.cur_obj or {}
local cur_obj = assert(state.cur_obj)
local txt = table.concat(state.doc_lines, '\n')
if cur_obj.desc then
cur_obj.desc = cur_obj.desc .. '\n' .. txt
else
cur_obj.desc = txt
end
state.doc_lines = nil
end
end
--- @param line string
--- @param state nvim.luacats.parser.State
local function process_doc_line(line, state)
line = line:sub(4):gsub('^%s+@', '@')
line = use_type_alt(line)
local parsed = luacats_grammar:match(line)
if not parsed then
if line:match('^ ') then
line = line:sub(2)
end
if state.last_doc_item then
if not state.last_doc_item_indent then
state.last_doc_item_indent = #line:match('^%s*') + 1
end
state.last_doc_item.desc = (state.last_doc_item.desc or '')
.. '\n'
.. line:sub(state.last_doc_item_indent or 1)
else
state.doc_lines = state.doc_lines or {}
table.insert(state.doc_lines, line)
end
return
end
state.last_doc_item_indent = nil
state.last_doc_item = nil
state.cur_obj = state.cur_obj or {}
local cur_obj = assert(state.cur_obj)
local kind = parsed.kind
if kind == 'brief' then
state.cur_obj = {
kind = 'brief',
desc = parsed.desc,
}
elseif kind == 'class' then
--- @cast parsed nvim.luacats.Class
state.cur_obj = {
kind = 'class',
name = parsed.name,
parent = parsed.parent,
desc = '',
fields = {},
}
elseif kind == 'field' then
--- @cast parsed nvim.luacats.Field
if not parsed.access then
parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil
if parsed.desc then
parsed.desc = vim.trim(parsed.desc)
end
table.insert(cur_obj.fields, parsed)
end
state.doc_lines = nil
elseif kind == 'param' then
state.last_doc_item_indent = nil
cur_obj.params = cur_obj.params or {}
if vim.endswith(parsed.name, '?') then
parsed.name = parsed.name:sub(1, -2)
parsed.type = parsed.type .. '?'
end
state.last_doc_item = {
name = parsed.name,
type = parsed.type,
desc = parsed.desc,
}
table.insert(cur_obj.params, state.last_doc_item)
elseif kind == 'return' then
cur_obj.returns = cur_obj.returns or {}
for _, t in ipairs(parsed) do
table.insert(cur_obj.returns, {
name = t.name,
type = t.type,
desc = parsed.desc,
})
end
state.last_doc_item_indent = nil
state.last_doc_item = cur_obj.returns[#cur_obj.returns]
elseif kind == 'private' then
cur_obj.access = 'private'
elseif kind == 'package' then
cur_obj.access = 'package'
elseif kind == 'protected' then
cur_obj.access = 'protected'
elseif kind == 'deprecated' then
cur_obj.deprecated = true
elseif kind == 'nodoc' then
cur_obj.nodoc = true
elseif kind == 'since' then
cur_obj.since = parsed.desc
elseif kind == 'see' then
cur_obj.see = cur_obj.see or {}
table.insert(cur_obj.see, { desc = parsed.desc })
elseif kind == 'note' then
state.last_doc_item_indent = nil
state.last_doc_item = {
desc = parsed.desc,
}
cur_obj.notes = cur_obj.notes or {}
table.insert(cur_obj.notes, state.last_doc_item)
elseif kind == 'type' then
cur_obj.desc = parsed.desc
parsed.desc = nil
parsed.kind = nil
cur_obj.type = parsed
elseif kind == 'alias' then
state.cur_obj = {
kind = 'alias',
desc = parsed.desc,
}
elseif kind == 'enum' then
-- TODO
state.doc_lines = nil
elseif
vim.tbl_contains({
'diagnostic',
'cast',
'overload',
'meta',
}, kind)
then
-- Ignore
return
elseif kind == 'generic' then
cur_obj.generics = cur_obj.generics or {}
cur_obj.generics[parsed.name] = parsed.type or 'any'
else
error('Unhandled' .. vim.inspect(parsed))
end
end
--- @param fun nvim.luacats.parser.fun
--- @return nvim.luacats.parser.field
local function fun2field(fun)
local parts = { 'fun(' }
for _, p in ipairs(fun.params or {}) do
parts[#parts + 1] = string.format('%s: %s', p.name, p.type)
end
parts[#parts + 1] = ')'
if fun.returns then
parts[#parts + 1] = ': '
local tys = {} --- @type string[]
for _, p in ipairs(fun.returns) do
tys[#tys + 1] = p.type
end
parts[#parts + 1] = table.concat(tys, ', ')
end
return {
name = fun.name,
type = table.concat(parts, ''),
access = fun.access,
desc = fun.desc,
}
end
--- Function to normalize known form for declaring functions and normalize into a more standard
--- form.
--- @param line string
--- @return string
local function filter_decl(line)
-- M.fun = vim._memoize(function(...)
-- ->
-- function M.fun(...)
line = line:gsub('^local (.+) = .*_memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)')
line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)')
return line
end
--- @param line string
--- @param state nvim.luacats.parser.State
--- @param classes table<string,nvim.luacats.parser.class>
--- @param classvars table<string,string>
--- @param has_indent boolean
local function process_lua_line(line, state, classes, classvars, has_indent)
line = filter_decl(line)
if state.cur_obj and state.cur_obj.kind == 'class' then
local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=')
if nm then
classvars[nm] = state.cur_obj.name
end
return
end
do
local parent_tbl, sep, fun_or_meth_nm =
line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(')
if parent_tbl then
-- Have a decl. Ensure cur_obj
state.cur_obj = state.cur_obj or {}
local cur_obj = assert(state.cur_obj)
-- Match `Class:foo` methods for defined classes
local class = classvars[parent_tbl]
if class then
--- @cast cur_obj nvim.luacats.parser.fun
cur_obj.name = fun_or_meth_nm
cur_obj.class = class
cur_obj.classvar = parent_tbl
-- Add self param to methods
if sep == ':' then
cur_obj.params = cur_obj.params or {}
table.insert(cur_obj.params, 1, {
name = 'self',
type = class,
})
end
-- Add method as the field to the class
table.insert(classes[class].fields, fun2field(cur_obj))
return
end
-- Match `M.foo`
if cur_obj and parent_tbl == cur_obj.modvar then
cur_obj.name = fun_or_meth_nm
return
end
end
end
do
-- Handle: `function A.B.C.foo(...)`
local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(')
if fn_nm then
state.cur_obj = state.cur_obj or {}
state.cur_obj.name = fn_nm
return
end
end
do
-- Handle: `M.foo = {...}` where `M` is the modvar
local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=')
if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then
state.cur_obj.name = tbl_nm
state.cur_obj.table = true
return
end
end
do
-- Handle: `foo = {...}`
local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=')
if tbl_nm and not has_indent then
state.cur_obj = state.cur_obj or {}
state.cur_obj.name = tbl_nm
state.cur_obj.table = true
return
end
end
do
-- Handle: `vim.foo = {...}`
local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=')
if state.cur_obj and tbl_nm and not has_indent then
state.cur_obj.name = tbl_nm
state.cur_obj.table = true
return
end
end
if state.cur_obj then
if line:find('^%s*%-%- luacheck:') then
state.cur_obj = nil
elseif line:find('^%s*local%s+') then
state.cur_obj = nil
elseif line:find('^%s*return%s+') then
state.cur_obj = nil
elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then
state.cur_obj = nil
end
end
end
--- Determine the table name used to export functions of a module
--- Usually this is `M`.
--- @param filename string
--- @return string?
local function determine_modvar(filename)
local modvar --- @type string?
for line in io.lines(filename) do
do
--- @type string?
local m = line:match('^return%s+([a-zA-Z_]+)')
if m then
modvar = m
end
end
do
--- @type string?
local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),')
if m then
modvar = m
end
end
end
return modvar
end
--- @param obj nvim.luacats.parser.obj
--- @param funs nvim.luacats.parser.fun[]
--- @param classes table<string,nvim.luacats.parser.class>
--- @param briefs string[]
--- @param uncommitted nvim.luacats.parser.obj[]
local function commit_obj(obj, classes, funs, briefs, uncommitted)
local commit = false
if obj.kind == 'class' then
--- @cast obj nvim.luacats.parser.class
if not classes[obj.name] then
classes[obj.name] = obj
commit = true
end
elseif obj.kind == 'alias' then
-- Just pretend
commit = true
elseif obj.kind == 'brief' then
--- @cast obj nvim.luacats.parser.brief`
briefs[#briefs + 1] = obj.desc
commit = true
else
--- @cast obj nvim.luacats.parser.fun`
if obj.name then
funs[#funs + 1] = obj
commit = true
end
end
if not commit then
table.insert(uncommitted, obj)
end
return commit
end
--- @param filename string
--- @param uncommitted nvim.luacats.parser.obj[]
-- luacheck: no unused
local function dump_uncommitted(filename, uncommitted)
local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt'
if #uncommitted > 0 then
print(string.format('Could not commit %d objects in %s', #uncommitted, filename))
vim.fn.mkdir(assert(vim.fs.dirname(out_path)), 'p')
local f = assert(io.open(out_path, 'w'))
for i, x in ipairs(uncommitted) do
f:write(i)
f:write(': ')
f:write(vim.inspect(x))
f:write('\n')
end
f:close()
else
vim.fn.delete(out_path)
end
end
local M = {}
--- @param filename string
--- @return table<string,nvim.luacats.parser.class> classes
--- @return nvim.luacats.parser.fun[] funs
--- @return string[] briefs
--- @return nvim.luacats.parser.obj[]
function M.parse(filename)
local funs = {} --- @type nvim.luacats.parser.fun[]
local classes = {} --- @type table<string,nvim.luacats.parser.class>
local briefs = {} --- @type string[]
local mod_return = determine_modvar(filename)
--- @type string
local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename
module = module:gsub('/', '.')
local classvars = {} --- @type table<string,string>
local state = {} --- @type nvim.luacats.parser.State
-- Keep track of any partial objects we don't commit
local uncommitted = {} --- @type nvim.luacats.parser.obj[]
for line in io.lines(filename) do
local has_indent = line:match('^%s+') ~= nil
line = vim.trim(line)
if vim.startswith(line, '---') then
process_doc_line(line, state)
else
add_doc_lines_to_obj(state)
if state.cur_obj then
state.cur_obj.modvar = mod_return
state.cur_obj.module = module
end
process_lua_line(line, state, classes, classvars, has_indent)
-- Commit the object
local cur_obj = state.cur_obj
if cur_obj then
if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then
--- @diagnostic disable-next-line:inject-field
cur_obj.line = line
end
end
state = {}
end
end
-- dump_uncommitted(filename, uncommitted)
return classes, funs, briefs, uncommitted
end
return M

239
scripts/text_utils.lua Normal file
View File

@@ -0,0 +1,239 @@
local fmt = string.format
--- @class nvim.text_utils.MDNode
--- @field [integer] nvim.text_utils.MDNode
--- @field type string
--- @field text? string
local INDENTATION = 4
local M = {}
local function contains(t, xs)
return vim.tbl_contains(xs, t)
end
--- @param text string
--- @return nvim.text_utils.MDNode
local function parse_md(text)
local parser = vim.treesitter.languagetree.new(text, 'markdown', {
injections = { markdown = '' },
})
local root = parser:parse(true)[1]:root()
local EXCLUDE_TEXT_TYPE = {
list = true,
list_item = true,
section = true,
document = true,
fenced_code_block = true,
fenced_code_block_delimiter = true,
}
--- @param node TSNode
--- @return nvim.text_utils.MDNode?
local function extract(node)
local ntype = node:type()
if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then
return
end
--- @type table<any,any>
local ret = { type = ntype }
if not EXCLUDE_TEXT_TYPE[ntype] then
ret.text = vim.treesitter.get_node_text(node, text)
end
for child, child_field in node:iter_children() do
local e = extract(child)
if child_field then
ret[child_field] = e
else
table.insert(ret, e)
end
end
return ret
end
return extract(root) or {}
end
--- @param x string
--- @param start_indent integer
--- @param indent integer
--- @param text_width integer
--- @return string
function M.wrap(x, start_indent, indent, text_width)
local words = vim.split(vim.trim(x), '%s+')
local parts = { string.rep(' ', start_indent) } --- @type string[]
local count = indent
for i, w in ipairs(words) do
if count > indent and count + #w > text_width - 1 then
parts[#parts + 1] = '\n'
parts[#parts + 1] = string.rep(' ', indent)
count = indent
elseif i ~= 1 then
parts[#parts + 1] = ' '
count = count + 1
end
count = count + #w
parts[#parts + 1] = w
end
return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', ''))
end
--- @param node nvim.text_utils.MDNode
--- @param start_indent integer
--- @param indent integer
--- @param text_width integer
--- @param level integer
--- @return string[]
local function render_md(node, start_indent, indent, text_width, level, is_list)
local parts = {} --- @type string[]
-- For debugging
local add_tag = false
-- local add_tag = true
if add_tag then
parts[#parts + 1] = '<' .. node.type .. '>'
end
if node.type == 'paragraph' then
local text = assert(node.text)
text = text:gsub('(%s)%*(%w+)%*(%s)', '%1%2%3')
text = text:gsub('(%s)_(%w+)_(%s)', '%1%2%3')
text = text:gsub('\\|', '|')
text = text:gsub('\\%*', '*')
text = text:gsub('\\_', '_')
parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width)
parts[#parts + 1] = '\n'
elseif node.type == 'code_fence_content' then
local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n')
local cindent = indent + INDENTATION
if level > 3 then
-- The tree-sitter markdown parser doesn't parse the code blocks indents
-- correctly in lists. Fudge it!
lines[1] = ' ' .. lines[1] -- ¯\_(ツ)_/¯
cindent = indent - level
local _, initial_indent = lines[1]:find('^%s*')
initial_indent = initial_indent + cindent
if initial_indent < indent then
cindent = indent - INDENTATION
end
end
for _, l in ipairs(lines) do
if #l > 0 then
parts[#parts + 1] = string.rep(' ', cindent)
parts[#parts + 1] = l
end
parts[#parts + 1] = '\n'
end
elseif node.type == 'fenced_code_block' then
parts[#parts + 1] = '>'
for _, child in ipairs(node) do
if child.type == 'info_string' then
parts[#parts + 1] = child.text
break
end
end
parts[#parts + 1] = '\n'
for _, child in ipairs(node) do
if child.type ~= 'info_string' then
vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1))
end
end
parts[#parts + 1] = '<\n'
elseif node.type == 'html_block' then
local text = node.text:gsub('^<pre>help', '')
text = text:gsub('</pre>%s*$', '')
parts[#parts + 1] = text
elseif node.type == 'list_marker_dot' then
parts[#parts + 1] = node.text
elseif contains(node.type, { 'list_marker_minus', 'list_marker_star' }) then
parts[#parts + 1] = '• '
elseif node.type == 'list_item' then
parts[#parts + 1] = string.rep(' ', indent)
local offset = node[1].type == 'list_marker_dot' and 3 or 2
for i, child in ipairs(node) do
local sindent = i <= 2 and 0 or (indent + offset)
vim.list_extend(
parts,
render_md(child, sindent, indent + offset, text_width, level + 1, true)
)
end
else
if node.text then
error(fmt('cannot render:\n%s', vim.inspect(node)))
end
for i, child in ipairs(node) do
vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1, is_list))
if node.type ~= 'list' and i ~= #node then
if (node[i + 1] or {}).type ~= 'list' then
parts[#parts + 1] = '\n'
end
end
end
end
if add_tag then
parts[#parts + 1] = '</' .. node.type .. '>'
end
return parts
end
--- @param text_width integer
local function align_tags(text_width)
--- @param line string
--- @return string
return function(line)
local tag_pat = '%s+(%*[^ ]+%*)%s*$'
local tags = {}
for m in line:gmatch(tag_pat) do
table.insert(tags, m)
end
if #tags > 0 then
line = line:gsub(tag_pat, '')
local tags_str = ' ' .. table.concat(tags, ' ')
local pad = string.rep(' ', text_width - #line - #tags_str)
return line .. pad .. tags_str
end
return line
end
end
--- @param text string
--- @param start_indent integer
--- @param indent integer
--- @param is_list? boolean
--- @return string
function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list)
-- Add an extra newline so the parser can properly capture ending ```
local parsed = parse_md(text .. '\n')
local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list)
local lines = vim.split(table.concat(ret), '\n')
lines = vim.tbl_map(align_tags(text_width), lines)
local s = table.concat(lines, '\n')
-- Reduce whitespace in code-blocks
s = s:gsub('\n+%s*>([a-z]+)\n?\n', ' >%1\n')
s = s:gsub('\n+%s*>\n?\n', ' >\n')
return s
end
return M