\n%s', text)
end
if opt.old then
-- XXX: Treat "old" docs as preformatted: they use indentation for layout.
-- Trim trailing newlines to avoid too much whitespace between divs.
return ('
%s
\n'):format(trim(text, 2))
end
return string.format('
\n%s\n
\n', text)
elseif node_name == 'line' then
if
(parent ~= 'codeblock' or parent ~= 'code')
and (is_blank(text) or is_noise(text, stats.noise_lines))
then
return '' -- Discard common "noise" lines.
end
-- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout.
local div = opt.old
and root:child(0)
and vim.list_contains({ 'column_heading', 'h1', 'h2', 'h3' }, root:child(0):type())
return string.format('%s%s', div and trim(text) or text, div and '' or '\n')
elseif parent == 'line_li' and node_name == 'prefix' then
return ''
elseif node_name == 'line_li' then
local prefix = first(root, 'prefix')
local numli = prefix and trim(node_text(prefix)):match('%d') -- Numbered listitem?
local sib = root:prev_sibling()
local prev_li = sib and sib:type() == 'line_li'
local cssclass = numli and 'help-li-num' or 'help-li'
-- Conjoin list items separated by blank lines (wrapped in separate blocks).
if not prev_li then
local parent_block = root:parent()
local prev_block = parent_block and parent_block:prev_sibling()
local prev_last = prev_block and prev_block:type() == 'block' and last(prev_block)
if prev_last and prev_last:type() == 'line_li' then
prev_li = true
sib = prev_last
end
end
if not prev_li then
opt.indent = 1
else
local sib_ws = ws(sib)
local this_ws = ws()
if get_indent(node_text()) == 0 then
opt.indent = 1
elseif this_ws > sib_ws then
-- Previous sibling is logically the _parent_ if it is indented less.
opt.indent = opt.indent + 1
elseif this_ws < sib_ws then
-- TODO(justinmk): This is buggy. Need to track exact whitespace length for each level.
opt.indent = math.max(1, opt.indent - 1)
end
end
local margin = opt.indent == 1 and '' or ('margin-left: %drem;'):format((1.5 * opt.indent))
return string.format('
%s
', cssclass, margin, text)
elseif node_name == 'taglink' or node_name == 'optionlink' then
local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname)
if ignored or not helppage then
return html_esc(node_text(root))
end
local s = ('%s
%s'):format(
ws(),
helppage,
url_encode(tagname),
html_esc(tagname)
)
if opt.old and node_name == 'taglink' then
s = fix_tab_after_conceal(s, node_text(root:next_sibling()))
end
return s
elseif vim.list_contains({ 'codespan', 'keycode' }, node_name) then
if root:has_error() then
return text
end
local s = ('%s
%s'):format(ws(), trimmed)
if opt.old and node_name == 'codespan' then
s = fix_tab_after_conceal(s, node_text(root:next_sibling()))
end
return s
elseif node_name == 'argument' then
return ('%s
%s'):format(ws(), trim(node_text(root)))
elseif node_name == 'codeblock' then
return text
elseif node_name == 'language' then
language = node_text(root)
return ''
elseif node_name == 'code' then -- Highlighted codeblock (child).
if is_blank(text) then
return ''
end
local code ---@type string
if language then
code = ('
%s
'):format(
language,
trim(trim_indent(text), 2)
)
language = nil
else
code = ('
%s
'):format(trim(trim_indent(text), 2))
end
return code
elseif node_name == 'tag' then -- anchor, h4 pseudo-heading
if root:has_error() then
return text
end
local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent)
local h4 = not in_heading and not next_ and get_indent(node_text()) > 8 -- h4 pseudo-heading
local cssclass = h4 and 'help-tag-right' or 'help-tag'
local tagname = node_text(root:child(1), false)
if vim.tbl_count(stats.first_tags) < 2 then
-- Force the first 2 tags in the doc to be anchored at the main heading.
table.insert(stats.first_tags, tagname)
return ''
end
local el = 'span'
local encoded_tagname = url_encode(tagname)
local s = ('%s<%s id="%s" class="%s">
%s%s>'):format(
ws(),
el,
encoded_tagname,
cssclass,
encoded_tagname,
trimmed,
el
)
if opt.old then
s = fix_tab_after_conceal(s, node_text(root:next_sibling()))
end
if in_heading and prev ~= 'tag' then
-- Start the
container for tags in a heading.
-- This makes "justify-content:space-between" right-align the tags.
-- foo bartag1 tag2
return string.format('%s', s)
elseif in_heading and next_ == nil then
-- End the container for tags in a heading.
return string.format('%s', s)
end
return s .. (h4 and '
' or '') -- HACK:
avoids h4 pseudo-heading mushing with text.
elseif node_name == 'delimiter' or node_name == 'modeline' then
return ''
elseif node_name == 'ERROR' then
if ignore_parse_error(opt.fname, trimmed) then
return text
end
-- Store the raw text to give context to the bug report.
local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]'
table.insert(stats.parse_errors, sample_text)
return ('%s'):format(
get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text),
trimmed
)
else -- Unknown token.
local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]'
return ('%s'):format(
node_name,
get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name),
trimmed
),
('unknown-token:"%s"'):format(node_name)
end
end
--- @param dir string e.g. '$VIMRUNTIME/doc'
--- @param include string[]|nil
--- @return string[]
local function get_helpfiles(dir, include)
local rv = {}
for f, type in vim.fs.dir(dir) do
if
vim.endswith(f, '.txt')
and type == 'file'
and (not include or vim.list_contains(include, f))
then
local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p')
table.insert(rv, fullpath)
end
end
return rv
end
--- Populates the helptags map.
--- @param help_dir string
--- @return table
local function _get_helptags(help_dir)
---@type table
local m = {}
-- Load a random help file to convince taglist() to do its job.
vim.cmd(string.format('split %s/api.txt', help_dir))
vim.cmd('lcd %:p:h')
local tags = vim.fn.taglist('.*') --[[ @as {name: string, filename: string}[] ]]
for _, item in ipairs(tags) do
if vim.endswith(item.filename, '.txt') then
m[item.name] = item.filename
end
end
vim.cmd('q!')
return m
end
--- Populates the helptags map.
local function get_helptags(help_dir)
local m = _get_helptags(help_dir)
--- XXX: Append tags from netrw, until we remove it...
local netrwtags = _get_helptags(vim.fs.normalize('$VIMRUNTIME/pack/dist/opt/netrw/doc/'))
m = vim.tbl_extend('keep', m, netrwtags)
return m
end
--- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system.
local function ensure_runtimepath()
if not vim.o.runtimepath:find('build/lib/nvim/') then
vim.cmd [[set runtimepath^=./build/lib/nvim/]]
end
end
--- Opens `fname` (or `text`, if given) in a buffer and gets a treesitter parser for the buffer contents.
---
--- @param fname string :help file to parse
--- @param text string? :help file contents
--- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr)
local function parse_buf(fname, text)
local buf ---@type integer
if text then
vim.cmd('split new') -- Text contents.
vim.api.nvim_put(vim.split(text, '\n'), '', false, false)
vim.cmd('setfiletype help')
buf = vim.api.nvim_get_current_buf()
elseif type(fname) == 'string' then
vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename.
buf = vim.api.nvim_get_current_buf()
else
-- Left for debugging
---@diagnostic disable-next-line: no-unknown
buf = fname
vim.cmd('sbuffer ' .. tostring(fname)) -- Buffer number.
end
local lang_tree = assert(vim.treesitter.get_parser(buf, nil))
lang_tree:parse()
return lang_tree, buf
end
--- Validates one :help file `fname`:
--- - checks that |tag| links point to valid helptags.
--- - recursively counts parse errors ("ERROR" nodes)
---
--- @param fname string help file to validate
--- @param request_urls boolean? whether to make requests to the URLs
--- @return { invalid_links: number, parse_errors: string[] }
local function validate_one(fname, request_urls)
local stats = {
parse_errors = {},
}
local lang_tree, buf = parse_buf(fname, nil)
for _, tree in ipairs(lang_tree:trees()) do
visit_validate(tree:root(), 0, tree, {
buf = buf,
fname = fname,
request_urls = request_urls,
}, stats)
end
lang_tree:destroy()
vim.cmd.close()
return stats
end
--- Generates HTML from one :help file `fname` and writes the result to `to_fname`.
---
--- @param fname string Source :help file.
--- @param text string|nil Source :help file contents, or nil to read `fname`.
--- @param to_fname string Destination .html file
--- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace)
---
--- @return string html
--- @return table stats
local function gen_one(fname, text, to_fname, old, commit)
local stats = {
noise_lines = {},
parse_errors = {},
first_tags = {}, -- Track the first few tags in doc.
}
local lang_tree, buf = parse_buf(fname, text)
---@type nvim.gen_help_html.heading[]
local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3.
local title = to_titlecase(basename_noext(fname))
local main = ''
for _, tree in ipairs(lang_tree:trees()) do
main = main
.. (
visit_node(
tree:root(),
0,
tree,
headings,
{ buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1 },
stats
)
)
end
local frontmatter = vim.json.encode({
title = title,
layout = 'single', -- Hugo-specific, to make _index.html the same as the other pages
aliases = { -- Hugo-specific, make /api.html redirect to /api/
vim.fs.joinpath('/doc/user', vim.fs.basename(to_fname)),
},
params = {
firstTag1 = stats.first_tags[1] or '',
firstTag2 = stats.first_tags[2] or '',
basename = vim.fs.basename(fname),
commit = commit,
parseErrors = #stats.parse_errors,
bugUrl = get_bug_url_nvim(fname, to_fname, 'TODO', nil),
noiseLines = html_esc(table.concat(stats.noise_lines, '\n')),
noiseLinesCount = #stats.noise_lines,
headings = headings,
},
}, { indent = ' ', sort_keys = true })
local html = ('%s\n%s'):format(frontmatter, main)
vim.cmd('q!')
lang_tree:destroy()
return html, stats
end
--- Generates a JSON map of tags to URL-encoded `filename#anchor` locations.
---
---@param fname string
local function gen_helptags_json(fname)
assert(tagmap, '`tagmap` not generated yet')
local t = {} ---@type table
for tag, f in pairs(tagmap) do
-- "foo.txt"
local helpfile = vim.fs.basename(f)
-- "foo.html"
local htmlpage = assert(get_helppage(helpfile))
-- "foo.html#tag"
t[tag] = ('%s#%s'):format(htmlpage, url_encode(tag))
end
tofile(fname, vim.json.encode(t, { indent = ' ', sort_keys = true }))
end
local function gen_helptag_html(fname)
local frontmatter = vim.json.encode({
title = 'Helptag redirect',
layout = 'helptag',
aliases = { vim.fs.basename(fname) },
}, { indent = ' ', sort_keys = true })
tofile(fname, frontmatter)
end
-- Testing
local function ok(cond, expected, actual, message)
assert(
(not expected and not actual) or (expected and actual),
'if "expected" is given, "actual" is also required'
)
if expected then
assert(
cond,
('%sexpected %s, got: %s'):format(
message and (message .. '\n') or '',
vim.inspect(expected),
vim.inspect(actual)
)
)
else
assert(cond)
end
return true
end
local function eq(expected, actual, message)
return ok(vim.deep_equal(expected, actual), expected, actual, message)
end
function M._test()
tagmap = get_helptags('$VIMRUNTIME/doc')
helpfiles = get_helpfiles(vim.fs.normalize('$VIMRUNTIME/doc'))
ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap))
ok(
vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'),
tagmap['vim.diagnostic.set()'],
'diagnostic.txt'
)
ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt')
ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]]))
ok(is_noise([[ NVIM REFERENCE MANUAL by Thiago de Arruda ]]))
ok(not is_noise([[vim:tw=78]]))
eq(0, get_indent('a'))
eq(1, get_indent(' a'))
eq(2, get_indent(' a\n b\n c\n'))
eq(5, get_indent(' a\n \n b\n c\n d\n e\n'))
eq(
'a\n \n b\n c\n d\n e\n',
trim_indent(' a\n \n b\n c\n d\n e\n')
)
local fixed_url, removed_chars = fix_url('https://example.com).')
eq('https://example.com', fixed_url)
eq(').', removed_chars)
fixed_url, removed_chars = fix_url('https://example.com.)')
eq('https://example.com.', fixed_url)
eq(')', removed_chars)
fixed_url, removed_chars = fix_url('https://example.com.')
eq('https://example.com', fixed_url)
eq('.', removed_chars)
fixed_url, removed_chars = fix_url('https://example.com)')
eq('https://example.com', fixed_url)
eq(')', removed_chars)
fixed_url, removed_chars = fix_url('https://example.com')
eq('https://example.com', fixed_url)
eq('', removed_chars)
print('all tests passed.\n')
end
--- @class nvim.gen_help_html.gen_result
--- @field helpfiles string[] list of generated HTML files, from the source docs {include}
--- @field err_count integer number of parse errors in :help docs
--- @field invalid_links table
--- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`.
---
--- Example:
---
--- gen('$VIMRUNTIME/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil)
---
--- @param help_dir string Source directory containing the :help files. Must run `make helptags` first.
--- @param to_dir string Target directory where the .html files will be written.
--- @param include string[]|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'}
--- @param commit string?
--- @param parser_path string? path to non-default vimdoc.so/dylib/dll
---
--- @return nvim.gen_help_html.gen_result result
function M.gen(help_dir, to_dir, include, commit, parser_path)
vim.validate('help_dir', help_dir, function(d)
return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
end, 'valid directory')
vim.validate('to_dir', to_dir, 'string')
vim.validate('include', include, 'table', true)
vim.validate('commit', commit, 'string', true)
vim.validate('parser_path', parser_path, function(f)
return vim.fn.filereadable(vim.fs.normalize(f)) == 1
end, true, 'valid vimdoc.{so,dll,dylib} filepath')
local err_count = 0
local redirects_count = 0
ensure_runtimepath()
parser_path = parser_path and vim.fs.normalize(parser_path) or nil
if parser_path then
-- XXX: Delete the installed .so files first, else this won't work :(
-- /usr/local/lib/nvim/parser/vimdoc.so
-- ./build/lib/nvim/parser/vimdoc.so
vim.treesitter.language.add('vimdoc', { path = parser_path })
end
tagmap = get_helptags(vim.fs.normalize(help_dir))
helpfiles = get_helpfiles(help_dir, include)
to_dir = vim.fs.normalize(to_dir)
print(('output dir: %s\n\n'):format(to_dir))
vim.fn.mkdir(to_dir, 'p')
-- NOTE: Better for Hugo to be in static/, but works fine with contents/ as to_dir
gen_helptags_json(('%s/helptags.json'):format(to_dir))
gen_helptag_html(('%s/helptag.html'):format(to_dir))
for _, f in ipairs(helpfiles) do
-- "foo.txt"
local helpfile = vim.fs.basename(f)
-- "to/dir/foo.html"
local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile, true))
local html, stats = gen_one(f, nil, to_fname, not new_layout[helpfile], commit or '?')
tofile(to_fname, html)
print(
('generated (%-2s errors): %-15s => %s'):format(
#stats.parse_errors,
helpfile,
vim.fs.basename(to_fname)
)
)
-- Generate redirect pages for renamed help files.
local helpfile_tag = (helpfile:gsub('%.txt$', '')):gsub('_', '-') -- "dev_tools.txt" => "dev-tools"
local redirect_from = redirects[helpfile]
if redirect_from then
local redirect_text = vim.text
.indent(
0,
[[
*%s* Nvim
Document moved to: |%s|
==============================================================================
Document moved
Document moved to: |%s|
==============================================================================
vim:tw=78:ts=8:ft=help:norl:
]]
)
:format(redirect_from, helpfile_tag, helpfile_tag, helpfile_tag, helpfile_tag, helpfile_tag)
local redirect_to = ('%s/%s'):format(to_dir, get_helppage(redirect_from, true))
local redirect_html, _ =
gen_one(redirect_from, redirect_text, redirect_to, false, commit or '?')
assert(
redirect_html:find(vim.pesc(helpfile_tag)),
('not found in redirect html: %s'):format(helpfile_tag)
)
tofile(redirect_to, redirect_html)
print(
('generated (redirect) : %-15s => %s'):format(
redirect_from .. '.txt',
vim.fs.basename(to_fname)
)
)
redirects_count = redirects_count + 1
end
err_count = err_count + #stats.parse_errors
end
print(('\ngenerated %d html pages'):format(#helpfiles + redirects_count))
print(('total errors: %d'):format(err_count))
-- Why aren't the netrw tags found in neovim/docs/ CI?
print(('invalid tags: %s'):format(vim.inspect(invalid_links)))
eq(redirects_count, include and redirects_count or vim.tbl_count(redirects)) -- sanity check
print(('redirects: %d'):format(redirects_count))
print('\n')
--- @type nvim.gen_help_html.gen_result
return {
helpfiles = helpfiles,
err_count = err_count,
invalid_links = invalid_links,
}
end
--- @class nvim.gen_help_html.validate_result
--- @field helpfiles integer number of generated helpfiles
--- @field err_count integer number of parse errors
--- @field parse_errors table
--- @field invalid_links table invalid tags in :help docs
--- @field invalid_urls table invalid URLs in :help docs
--- @field invalid_spelling table> invalid spelling in :help docs
--- Validates all :help files found in `help_dir`:
--- - checks that |tag| links point to valid helptags.
--- - recursively counts parse errors ("ERROR" nodes)
---
--- This is 10x faster than gen(), for use in CI.
---
--- @return nvim.gen_help_html.validate_result result
function M.validate(help_dir, include, parser_path, request_urls)
vim.validate('help_dir', help_dir, function(d)
return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
end, 'valid directory')
vim.validate('include', include, 'table', true)
vim.validate('parser_path', parser_path, function(f)
return vim.fn.filereadable(vim.fs.normalize(f)) == 1
end, true, 'valid vimdoc.{so,dll,dylib} filepath')
local err_count = 0 ---@type integer
local files_to_errors = {} ---@type table
ensure_runtimepath()
parser_path = parser_path and vim.fs.normalize(parser_path) or nil
if parser_path then
-- XXX: Delete the installed .so files first, else this won't work :(
-- /usr/local/lib/nvim/parser/vimdoc.so
-- ./build/lib/nvim/parser/vimdoc.so
vim.treesitter.language.add('vimdoc', { path = parser_path })
end
tagmap = get_helptags(vim.fs.normalize(help_dir))
helpfiles = get_helpfiles(help_dir, include)
for _, f in ipairs(helpfiles) do
local helpfile = vim.fs.basename(f)
local rv = validate_one(f, request_urls)
print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile))
if #rv.parse_errors > 0 then
files_to_errors[helpfile] = rv.parse_errors
vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v)
return s .. '\n ' .. v
end)))
end
err_count = err_count + #rv.parse_errors
end
-- Requests are async, wait for them to finish.
-- TODO(yochem): `:cancel()` tasks after #36146
vim.wait(20000, function()
return pending_urls <= 0
end)
ok(pending_urls <= 0, 'pending url checks', pending_urls)
---@type nvim.gen_help_html.validate_result
return {
helpfiles = #helpfiles,
err_count = err_count,
parse_errors = files_to_errors,
invalid_links = invalid_links,
invalid_urls = invalid_urls,
invalid_spelling = invalid_spelling,
}
end
--- Validates vimdoc files in $VIMRUNTIME, and prints error messages on failure.
---
--- If this fails, try these steps (in order):
--- 1. Fix/cleanup the :help docs.
--- 2. Fix the parser: https://github.com/neovim/tree-sitter-vimdoc
--- 3. File a parser bug, and adjust the tolerance of this test in the meantime.
---
--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
--- @param request_urls? boolean make network requests to check if the URLs are reachable.
function M.run_validate(help_dir, request_urls)
help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
print('doc path = ' .. vim.uv.fs_realpath(help_dir))
local rv = M.validate(help_dir, nil, nil, request_urls)
-- Check that we actually found helpfiles.
ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles)
eq({}, rv.parse_errors, 'no parse errors')
eq(0, rv.err_count, 'no parse errors')
eq({}, rv.invalid_links, 'invalid tags in :help docs')
eq({}, rv.invalid_urls, 'invalid URLs in :help docs')
eq(
{},
rv.invalid_spelling,
'invalid spelling in :help docs (see spell_dict in src/gen/gen_help_html.lua)'
)
end
--- Test-generates HTML from docs.
---
--- 1. Test that gen_help_html.lua actually works.
--- 2. Test that parse errors did not increase wildly. Because we explicitly test only a few
--- :help files, we can be precise about the tolerances here.
--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
function M.test_gen(help_dir)
local tmpdir = vim.fs.dirname(vim.fn.tempname())
help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
print('doc path = ' .. vim.uv.fs_realpath(help_dir))
-- Because gen() is slow (~30s), this test is limited to a few files.
local input = { 'api.txt', 'index.txt', 'nvim.txt' }
local rv = M.gen(help_dir, tmpdir, input)
eq(#input, #rv.helpfiles)
eq(0, rv.err_count, 'parse errors in :help docs')
eq({}, rv.invalid_links, 'invalid tags in :help docs')
end
return M