feat(help): generate :helptags using Treesitter

Problem:
Tags are manually parsed in C which is not flexible and prone to errors.
Extending the help system to allow for other formats (e.g. Markdown)
would require a large rewrite in the C core, while with Treesitter it
only needs a query update.

Solution:
Use the power of treesitter to extract the tags from helpfiles.

- build: set `$VIMRUNTIME` when generating helptags, like
  `cmake/Util.cmake` already does for other generators.
- fix(help): only accept tags delimited by whitespace. The old C parser
  only accepted a `*tag*` preceded by start-of-line or whitespace and
  followed by whitespace or end-of-line. The vimdoc parser also captures
  tags followed by other text, e.g. `*$XDG_STATE_HOME*/.../logs` in
  starting.txt, which caused an E154 duplicate tag error for docs that
  were previously fine.
This commit is contained in:
Yochem van Rosmalen
2026-01-14 12:03:21 +01:00
committed by Justin M. Keyes
parent a1de07418b
commit b36b3d7f3a
5 changed files with 243 additions and 304 deletions

View File

@@ -41,7 +41,7 @@ foreach(PACKAGE ${PACKAGES})
add_custom_command(OUTPUT "${GENERATED_PACKAGE_DIR}/${PACKNAME}/doc/tags"
COMMAND ${CMAKE_COMMAND} -E copy_directory
${PACKAGE} ${GENERATED_PACKAGE_DIR}/${PACKNAME}
COMMAND ${NVIM_HOST_PRG}
COMMAND ${CMAKE_COMMAND} -E env "VIMRUNTIME=${PROJECT_SOURCE_DIR}/runtime" ${NVIM_HOST_PRG}
-u NONE -i NONE -e --headless -c "helptags doc" -c quit
DEPENDS
nvim_bin
@@ -73,7 +73,7 @@ add_custom_command(OUTPUT ${GENERATED_HELP_TAGS}
COMMAND ${CMAKE_COMMAND} -E remove_directory doc
COMMAND ${CMAKE_COMMAND} -E copy_directory
${PROJECT_SOURCE_DIR}/runtime/doc doc
COMMAND ${NVIM_HOST_PRG}
COMMAND ${CMAKE_COMMAND} -E env "VIMRUNTIME=${PROJECT_SOURCE_DIR}/runtime" ${NVIM_HOST_PRG}
-u NONE -i NONE -e --headless -c "helptags ++t doc" -c quit
DEPENDS
nvim_bin

View File

@@ -529,6 +529,9 @@ These existing features changed their behavior.
• |undo| no longer restores the old (wrong) position of a mark that you moved
(|m|, |:mark|) later. The mark shifts with the text it was moved to, the
same as a mark the change never touched.
• |:helptags| finds help tags with the "vimdoc" |treesitter| parser, and
`:helptags ALL` reports |E152| for "doc" directories it cannot write,
instead of silently skipping them.
==============================================================================
REMOVED FEATURES *news-removed*

View File

@@ -1,5 +1,7 @@
local M = {}
local ts = vim.treesitter
local tag_exceptions = {
-- Interpret asterisk (star, '*') literal but name it 'star'
['*'] = 'star',
@@ -395,4 +397,149 @@ function M.local_additions()
end
end
local query = ts.query.parse('vimdoc', '(tag (word) @tagname)')
--- @alias Tag { [1]: string, [2]: string, [3]: string} tuple of tag, file, and search command
---Find and report duplicate tags.
---@param tags Tag[]
---@return boolean
local function find_duplicates(tags)
local prevtag, prevfn, has_duplicates = '', '', false
for _, tagline in ipairs(tags) do
local curtag, curfn, _ = unpack(tagline)
if curtag == prevtag then
has_duplicates = true
local filenames = prevfn ~= curfn and (curfn .. ' and ' .. prevfn) or curfn
local msg = ('E154: Duplicate tag "%s" in %s'):format(curtag, filenames)
vim.api.nvim_echo({ { msg } }, false, { err = true })
end
prevtag = curtag
prevfn = curfn
end
return has_duplicates
end
---Extract tags from {file} and add to list of tags. Modifies {tags}.
---@param tags Tag[]
---@param file string
local function extract_tags(tags, file)
local filename = vim.fs.basename(file)
local source = vim.fn.readblob(file)
local parser = ts.get_string_parser(source, 'vimdoc')
local tree = assert(parser:parse())
local root = tree[1]:root()
for _, match in query:iter_matches(root, source) do
for id, node in pairs(match) do
if query.captures[id] == 'tagname' then
-- Only accept a *tag* when there is white space (or nothing) before it
-- and it is followed by a white character or end-of-line.
local _, _, start_byte, _, _, end_byte = node[1]:parent():range(true)
local before = source:sub(start_byte, start_byte)
local after = source:sub(end_byte + 1, end_byte + 1)
if before:match('^[ \t\n\r]?$') and after:match('^[ \t\n\r]?$') then
local tagname = ts.get_node_text(node[1], source)
local escaped = tagname:gsub('[\\/]', '\\%0')
local searchcmd = '/*' .. escaped .. '*'
table.insert(tags, { tagname, filename, searchcmd })
end
end
end
end
end
--- Extract tags from helpfiles and combine in a single 'tags' file.
--- @param helpfiles string[] list of helpfiles
--- @param outpath string path to write the 'tags' file to.
--- @param include_helptags_tag boolean true if the 'help-tags' tag should be included
local function gen_tagsfile(helpfiles, outpath, include_helptags_tag)
---@type Tag[]
local tags = {}
-- (1) extract tags from all files
for _, file in ipairs(helpfiles) do
extract_tags(tags, file)
end
if include_helptags_tag then
table.insert(tags, { 'help-tags', 'tags', '1' })
end
if vim.tbl_isempty(tags) then
return
end
-- (2) sort alphabetically on tag name
table.sort(tags, function(a, b)
return a[1] < b[1]
end)
-- (3) check duplicates
local has_duplicates = find_duplicates(tags)
-- (4) write tags to file
local f = assert(io.open(outpath, 'w'))
for _, tag in ipairs(tags) do
f:write(table.concat(tag, '\t') .. '\n')
end
f:close()
-- tags file has to be written before we can error
if has_duplicates then
error('duplicate tags')
end
end
--- Create a "tags" file for all help files in the given directory.
---
--- The directory {dir} is generally a "doc" directory that contains "*.txt"
--- helpfiles.
---
--- @param dir string? Path to directory with help files. If `nil` (or |vim.NIL|),
--- generate tags for every `doc` directory in the runtimepath.
--- @param include_index_tag? boolean (default: false) Whether to include the "help-tags" tag.
function M.gen_tags(dir, include_index_tag)
if dir == vim.NIL then
dir = nil
end
vim.validate('dir', dir, 'string', true)
vim.validate('include_index_tag', include_index_tag, 'boolean', true)
local dirs = dir and { vim.fs.normalize(dir) } or vim.api.nvim_get_runtime_file('doc', true)
local vimruntime = vim.fs.normalize(vim.fs.joinpath(vim.env.VIMRUNTIME, 'doc'))
for _, directory in ipairs(dirs) do
local files = vim.fs.find(function(name, _)
return vim.endswith(name, '.txt')
end, { path = directory, type = 'file', limit = math.huge })
local outpath = vim.fs.joinpath(directory, 'tags')
gen_tagsfile(files, outpath, include_index_tag or directory == vimruntime)
-- handle translated help files per language
local translated = vim.fs.find(function(name, _)
-- "*.[a-z][a-z]x", see :help help-translated
return name:match('%.%l%lx', -4)
end, { path = directory, type = 'file', limit = math.huge })
-- categorize translated files per two-letter language code
---@type table<string, string[]>
local per_lang = {}
for _, file in ipairs(translated) do
-- extract language code "nl" from filename "plugin.nlx"
local lang = file:sub(-3, -2)
per_lang[lang] = per_lang[lang] or {}
table.insert(per_lang[lang], file)
end
for lang, langfiles in pairs(per_lang) do
local tagsfile = vim.fs.joinpath(directory, 'tags-' .. lang)
gen_tagsfile(langfiles, tagsfile, include_index_tag or directory == vimruntime)
end
end
end
return M

View File

@@ -476,316 +476,24 @@ void ex_viusage(exarg_T *eap)
do_cmdline_cmd("help normal-index");
}
/// Generate tags in one help directory
///
/// @param dir Path to the doc directory
/// @param ext Suffix of the help files (".txt", ".itx", ".frx", etc.)
/// @param tagname Name of the tags file ("tags" for English, "tags-fr" for
/// French)
/// @param add_help_tags Whether to add the "help-tags" tag
/// @param ignore_writeerr ignore write error
static void helptags_one(char *dir, const char *ext, const char *tagfname, bool add_help_tags,
bool ignore_writeerr)
FUNC_ATTR_NONNULL_ALL
{
garray_T ga;
int filecount;
char **files;
char *s;
// Find all *.txt files.
size_t dirlen = xstrlcpy(NameBuff, dir, sizeof(NameBuff));
if (dirlen >= MAXPATHL
|| xstrlcat(NameBuff, "/**/*", sizeof(NameBuff)) >= MAXPATHL // NOLINT
|| xstrlcat(NameBuff, ext, sizeof(NameBuff)) >= MAXPATHL) {
emsg(_(e_fnametoolong));
return;
}
// Note: We cannot just do `&NameBuff` because it is a statically sized array
// so `NameBuff == &NameBuff` according to C semantics.
char *buff_list[1] = { NameBuff };
const int res = gen_expand_wildcards(1, buff_list, &filecount, &files,
EW_FILE|EW_SILENT);
if (res == FAIL || filecount == 0) {
if (!got_int) {
semsg(_("E151: No match: %s"), NameBuff);
}
if (res != FAIL) {
FreeWild(filecount, files);
}
return;
}
// Open the tags file for writing.
// Do this before scanning through all the files.
memcpy(NameBuff, dir, dirlen + 1);
if (!add_pathsep(NameBuff)
|| xstrlcat(NameBuff, tagfname, sizeof(NameBuff)) >= MAXPATHL) {
emsg(_(e_fnametoolong));
return;
}
FILE *const fd_tags = os_fopen(NameBuff, "w");
if (fd_tags == NULL) {
if (!ignore_writeerr) {
semsg(_("E152: Cannot open %s for writing"), NameBuff);
}
FreeWild(filecount, files);
return;
}
// If using the "++t" argument or generating tags for "$VIMRUNTIME/doc"
// add the "help-tags" tag.
ga_init(&ga, (int)sizeof(char *), 100);
if (add_help_tags
|| path_equal("$VIMRUNTIME/doc", dir, kPathCmpExpand)) {
size_t s_len = 18 + strlen(tagfname);
s = xmalloc(s_len);
snprintf(s, s_len, "help-tags\t%s\t1\n", tagfname);
GA_APPEND(char *, &ga, s);
}
// Go over all the files and extract the tags.
for (int fi = 0; fi < filecount && !got_int; fi++) {
FILE *const fd = os_fopen(files[fi], "r");
if (fd == NULL) {
semsg(_("E153: Unable to open %s for reading"), files[fi]);
continue;
}
const char *const fname = files[fi] + dirlen + 1;
bool in_example = false;
while (!vim_fgets(IObuff, IOSIZE, fd) && !got_int) {
if (in_example) {
// skip over example; a non-white in the first column ends it
if (vim_strchr(" \t\n\r", (uint8_t)IObuff[0])) {
continue;
}
in_example = false;
}
char *p1 = vim_strchr(IObuff, '*'); // find first '*'
while (p1 != NULL) {
char *p2 = strchr(p1 + 1, '*'); // Find second '*'.
if (p2 != NULL && p2 > p1 + 1) { // Skip "*" and "**".
for (s = p1 + 1; s < p2; s++) {
if (*s == ' ' || *s == '\t' || *s == '|') {
break;
}
}
// Only accept a *tag* when it consists of valid
// characters, there is white space before it and is
// followed by a white character or end-of-line.
if (s == p2
&& (p1 == IObuff || p1[-1] == ' ' || p1[-1] == '\t')
&& (vim_strchr(" \t\n\r", (uint8_t)s[1]) != NULL
|| s[1] == NUL)) {
*p2 = NUL;
p1++;
size_t s_len = (size_t)(p2 - p1) + strlen(fname) + 2;
s = xmalloc(s_len);
GA_APPEND(char *, &ga, s);
snprintf(s, s_len, "%s\t%s", p1, fname);
// find next '*'
p2 = vim_strchr(p2 + 1, '*');
}
}
p1 = p2;
}
size_t off = strlen(IObuff);
if (off >= 2 && IObuff[off - 1] == '\n') {
off -= 2;
while (off > 0 && (ASCII_ISLOWER(IObuff[off]) || ascii_isdigit(IObuff[off]))) {
off--;
}
if (IObuff[off] == '>' && (off == 0 || IObuff[off - 1] == ' ')) {
in_example = true;
}
}
line_breakcheck();
}
fclose(fd);
}
FreeWild(filecount, files);
if (!got_int && ga.ga_data != NULL) {
// Sort the tags.
sort_strings(ga.ga_data, ga.ga_len);
// Check for duplicates.
for (int i = 1; i < ga.ga_len; i++) {
char *p1 = ((char **)ga.ga_data)[i - 1];
char *p2 = ((char **)ga.ga_data)[i];
while (*p1 == *p2) {
if (*p2 == '\t') {
*p2 = NUL;
vim_snprintf(NameBuff, MAXPATHL,
_("E154: Duplicate tag \"%s\" in file %s/%s"),
((char **)ga.ga_data)[i], dir, p2 + 1);
emsg(NameBuff);
*p2 = '\t';
break;
}
p1++;
p2++;
}
}
// Write the tags into the file.
for (int i = 0; i < ga.ga_len; i++) {
s = ((char **)ga.ga_data)[i];
if (strncmp(s, "help-tags\t", 10) == 0) {
// help-tags entry was added in formatted form
fputs(s, fd_tags);
} else {
fprintf(fd_tags, "%s\t/" "*", s);
for (char *p1 = s; *p1 != '\t'; p1++) {
// insert backslash before '\\' and '/'
if (*p1 == '\\' || *p1 == '/') {
putc('\\', fd_tags);
}
putc(*p1, fd_tags);
}
fprintf(fd_tags, "*\n");
}
}
}
GA_DEEP_CLEAR_PTR(&ga);
fclose(fd_tags); // there is no check for an error...
}
/// Generate tags in one help directory, taking care of translations.
static void do_helptags(char *dirname, bool add_help_tags, bool ignore_writeerr)
FUNC_ATTR_NONNULL_ALL
{
garray_T ga;
char lang[2];
char ext[5];
char fname[8];
int filecount;
char **files;
// Get a list of all files in the help directory and in subdirectories.
xstrlcpy(NameBuff, dirname, sizeof(NameBuff));
if (!add_pathsep(NameBuff)
|| xstrlcat(NameBuff, "**", sizeof(NameBuff)) >= MAXPATHL) {
emsg(_(e_fnametoolong));
return;
}
// Note: We cannot just do `&NameBuff` because it is a statically sized array
// so `NameBuff == &NameBuff` according to C semantics.
char *buff_list[1] = { NameBuff };
if (gen_expand_wildcards(1, buff_list, &filecount, &files,
EW_FILE|EW_SILENT) == FAIL
|| filecount == 0) {
semsg(_("E151: No match: %s"), NameBuff);
return;
}
// Go over all files in the directory to find out what languages are
// present.
int j;
ga_init(&ga, 1, 10);
for (int i = 0; i < filecount; i++) {
int len = (int)strlen(files[i]);
if (len <= 4) {
continue;
}
if (STRICMP(files[i] + len - 4, ".txt") == 0) {
// ".txt" -> language "en"
lang[0] = 'e';
lang[1] = 'n';
} else if (files[i][len - 4] == '.'
&& ASCII_ISALPHA(files[i][len - 3])
&& ASCII_ISALPHA(files[i][len - 2])
&& TOLOWER_ASC(files[i][len - 1]) == 'x') {
// ".abx" -> language "ab"
lang[0] = (char)TOLOWER_ASC(files[i][len - 3]);
lang[1] = (char)TOLOWER_ASC(files[i][len - 2]);
} else {
continue;
}
// Did we find this language already?
for (j = 0; j < ga.ga_len; j += 2) {
if (strncmp(lang, ((char *)ga.ga_data) + j, 2) == 0) {
break;
}
}
if (j == ga.ga_len) {
// New language, add it.
ga_grow(&ga, 2);
((char *)ga.ga_data)[ga.ga_len++] = lang[0];
((char *)ga.ga_data)[ga.ga_len++] = lang[1];
}
}
// Loop over the found languages to generate a tags file for each one.
for (j = 0; j < ga.ga_len; j += 2) {
STRCPY(fname, "tags-xx");
fname[5] = ((char *)ga.ga_data)[j];
fname[6] = ((char *)ga.ga_data)[j + 1];
if (fname[5] == 'e' && fname[6] == 'n') {
// English is an exception: use ".txt" and "tags".
fname[4] = NUL;
STRCPY(ext, ".txt");
} else {
// Language "ab" uses ".abx" and "tags-ab".
STRCPY(ext, ".xxx");
ext[1] = fname[5];
ext[2] = fname[6];
}
helptags_one(dirname, ext, fname, add_help_tags, ignore_writeerr);
}
ga_clear(&ga);
FreeWild(filecount, files);
}
static bool helptags_cb(int num_fnames, char **fnames, bool all, void *cookie)
FUNC_ATTR_NONNULL_ALL
{
for (int i = 0; i < num_fnames; i++) {
do_helptags(fnames[i], *(bool *)cookie, true);
if (!all) {
return true;
}
}
return num_fnames > 0;
}
/// ":helptags"
void ex_helptags(exarg_T *eap)
{
expand_T xpc;
bool add_help_tags = false;
// Check for ":helptags ++t {dir}".
// Check for ++t in ":helptags ++t {dir}".
if (strncmp(eap->arg, "++t", 3) == 0 && ascii_iswhite(eap->arg[3])) {
add_help_tags = true;
eap->arg = skipwhite(eap->arg + 3);
}
typval_T tv_args[] = {
{ .v_type = VAR_STRING, .vval.v_string = eap->arg },
{ .v_type = VAR_BOOL, .vval.v_bool = add_help_tags ? kBoolVarTrue : kBoolVarFalse },
{ .v_type = VAR_UNKNOWN },
};
if (strcmp(eap->arg, "ALL") == 0) {
do_in_path(p_rtp, "", "doc", DIP_ALL + DIP_DIR, helptags_cb, &add_help_tags);
} else {
ExpandInit(&xpc);
xpc.xp_context = EXPAND_DIRECTORIES;
char *dirname =
ExpandOne(&xpc, eap->arg, NULL, WILD_LIST_NOTFOUND|WILD_SILENT, WILD_EXPAND_FREE);
if (dirname == NULL || !os_isdir(dirname)) {
semsg(_("E150: Not a directory: %s"), eap->arg);
} else {
do_helptags(dirname, add_help_tags, false);
}
xfree(dirname);
tv_args[0] = (typval_T){ .v_type = VAR_SPECIAL, .vval.v_special = kSpecialVarNull };
}
nlua_call_typval("vim._core.help", "gen_tags", tv_args, NULL);
}

View File

@@ -1,11 +1,12 @@
local t = require('test.testutil')
local n = require('test.functional.testnvim')()
local describe, it, before_each, finally = t.describe, t.it, t.before_each, t.finally
local describe, it, before_each, after_each, finally =
t.describe, t.it, t.before_each, t.after_each, t.finally
local clear = n.clear
local eval = n.eval
local command = n.command
local eq = t.eq
local pcall_err = t.pcall_err
local fn = n.fn
local api = n.api
local mkdir = t.mkdir
@@ -324,3 +325,83 @@ describe(':help', function()
eq('*…*', api.nvim_get_current_line())
end)
end)
describe(':helptags', function()
before_each(function()
for _, sfx in ipairs({ '', '2' }) do
fn.mkdir(('Xhelptags%s/doc'):format(sfx), 'p')
for _, tag in ipairs({ 'Xa', 'Xb' }) do
write_file(('Xhelptags%s/doc/%s%s.txt'):format(sfx, tag, sfx), ('*%s%s*'):format(tag, sfx))
end
end
clear()
command('set rtp+=Xhelptags,Xhelptags2')
end)
after_each(function()
rmdir('Xhelptags')
rmdir('Xhelptags2')
end)
it('requires an argument', function()
local msg = t.pcall_err(command, 'helptags')
eq(true, msg:find('E471') ~= nil)
end)
it('{dir}', function()
command('helptags Xhelptags/doc')
eq(eval("['Xa Xa.txt /*Xa*','Xb Xb.txt /*Xb*']"), eval("readfile('Xhelptags/doc/tags')"))
command('help Xa')
eq('*Xa*', api.nvim_get_current_line())
end)
it('ALL', function()
command('helptags ALL')
eq(eval("['Xa Xa.txt /*Xa*','Xb Xb.txt /*Xb*']"), eval("readfile('Xhelptags/doc/tags')"))
eq(eval("['Xa2 Xa2.txt /*Xa2*','Xb2 Xb2.txt /*Xb2*']"), eval("readfile('Xhelptags2/doc/tags')"))
command('help Xa2')
eq('*Xa2*', api.nvim_get_current_line())
end)
it('++t', function()
command('helptags ++t Xhelptags/doc')
eq('help-tags tags 1', eval("readfile('Xhelptags/doc/tags')[-1]"))
end)
it('generates help-tag tag for VIMRUNTIME', function()
command('let $VIMRUNTIME="Xhelptags"')
command('helptags Xhelptags/doc')
eq('help-tags tags 1', eval("readfile('Xhelptags/doc/tags')[-1]"))
end)
it('errors on duplicate tags', function()
-- duplicate tags in different files
write_file('Xhelptags/doc/Xd.txt', '*Xa*', nil, true)
local msg = t.pcall_err(command, 'helptags Xhelptags/doc')
eq(true, msg:find('E154') ~= nil)
-- tags file should still be generated
eq(1, eval("filereadable('Xhelptags/doc/tags')"))
os.remove('Xhelptags/doc/Xd.txt')
-- duplicate tags in same file
write_file('Xhelptags/doc/Xa.txt', '\n*Xa*', nil, true)
msg = t.pcall_err(command, 'helptags Xhelptags/doc')
eq(true, msg:find('E154') ~= nil)
eq(1, eval("filereadable('Xhelptags/doc/tags')"))
end)
it('with translated help files', function()
write_file('Xhelptags/doc/Xa.nlx', '*Xa*', nil, true)
command('helptags Xhelptags/doc')
eq(1, eval("filereadable('Xhelptags/doc/tags-nl')"))
end)
end)