From b36b3d7f3a07cfcc4e76079188bfd7d3976a9cd3 Mon Sep 17 00:00:00 2001 From: Yochem van Rosmalen Date: Wed, 14 Jan 2026 12:03:21 +0100 Subject: [PATCH] feat(help): generate :helptags using Treesitter Problem: Tags are manually parsed in C which is not flexible and prone to errors. Extending the help system to allow for other formats (e.g. Markdown) would require a large rewrite in the C core, while with Treesitter it only needs a query update. Solution: Use the power of treesitter to extract the tags from helpfiles. - build: set `$VIMRUNTIME` when generating helptags, like `cmake/Util.cmake` already does for other generators. - fix(help): only accept tags delimited by whitespace. The old C parser only accepted a `*tag*` preceded by start-of-line or whitespace and followed by whitespace or end-of-line. The vimdoc parser also captures tags followed by other text, e.g. `*$XDG_STATE_HOME*/.../logs` in starting.txt, which caused an E154 duplicate tag error for docs that were previously fine. --- runtime/CMakeLists.txt | 4 +- runtime/doc/news.txt | 3 + runtime/lua/vim/_core/help.lua | 147 ++++++++++++ src/nvim/help.c | 308 +------------------------- test/functional/ex_cmds/help_spec.lua | 85 ++++++- 5 files changed, 243 insertions(+), 304 deletions(-) diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 7e8f445512..da87d13259 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -41,7 +41,7 @@ foreach(PACKAGE ${PACKAGES}) add_custom_command(OUTPUT "${GENERATED_PACKAGE_DIR}/${PACKNAME}/doc/tags" COMMAND ${CMAKE_COMMAND} -E copy_directory ${PACKAGE} ${GENERATED_PACKAGE_DIR}/${PACKNAME} - COMMAND ${NVIM_HOST_PRG} + COMMAND ${CMAKE_COMMAND} -E env "VIMRUNTIME=${PROJECT_SOURCE_DIR}/runtime" ${NVIM_HOST_PRG} -u NONE -i NONE -e --headless -c "helptags doc" -c quit DEPENDS nvim_bin @@ -73,7 +73,7 @@ add_custom_command(OUTPUT ${GENERATED_HELP_TAGS} COMMAND ${CMAKE_COMMAND} -E remove_directory doc COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/runtime/doc doc - COMMAND ${NVIM_HOST_PRG} + COMMAND ${CMAKE_COMMAND} -E env "VIMRUNTIME=${PROJECT_SOURCE_DIR}/runtime" ${NVIM_HOST_PRG} -u NONE -i NONE -e --headless -c "helptags ++t doc" -c quit DEPENDS nvim_bin diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt index 50d163b8de..85cc09d546 100644 --- a/runtime/doc/news.txt +++ b/runtime/doc/news.txt @@ -529,6 +529,9 @@ These existing features changed their behavior. • |undo| no longer restores the old (wrong) position of a mark that you moved (|m|, |:mark|) later. The mark shifts with the text it was moved to, the same as a mark the change never touched. +• |:helptags| finds help tags with the "vimdoc" |treesitter| parser, and + `:helptags ALL` reports |E152| for "doc" directories it cannot write, + instead of silently skipping them. ============================================================================== REMOVED FEATURES *news-removed* diff --git a/runtime/lua/vim/_core/help.lua b/runtime/lua/vim/_core/help.lua index 0dad8f93ff..efe57a9bcd 100644 --- a/runtime/lua/vim/_core/help.lua +++ b/runtime/lua/vim/_core/help.lua @@ -1,5 +1,7 @@ local M = {} +local ts = vim.treesitter + local tag_exceptions = { -- Interpret asterisk (star, '*') literal but name it 'star' ['*'] = 'star', @@ -395,4 +397,149 @@ function M.local_additions() end end +local query = ts.query.parse('vimdoc', '(tag (word) @tagname)') + +--- @alias Tag { [1]: string, [2]: string, [3]: string} tuple of tag, file, and search command + +---Find and report duplicate tags. +---@param tags Tag[] +---@return boolean +local function find_duplicates(tags) + local prevtag, prevfn, has_duplicates = '', '', false + + for _, tagline in ipairs(tags) do + local curtag, curfn, _ = unpack(tagline) + if curtag == prevtag then + has_duplicates = true + local filenames = prevfn ~= curfn and (curfn .. ' and ' .. prevfn) or curfn + local msg = ('E154: Duplicate tag "%s" in %s'):format(curtag, filenames) + vim.api.nvim_echo({ { msg } }, false, { err = true }) + end + prevtag = curtag + prevfn = curfn + end + + return has_duplicates +end + +---Extract tags from {file} and add to list of tags. Modifies {tags}. +---@param tags Tag[] +---@param file string +local function extract_tags(tags, file) + local filename = vim.fs.basename(file) + local source = vim.fn.readblob(file) + local parser = ts.get_string_parser(source, 'vimdoc') + + local tree = assert(parser:parse()) + local root = tree[1]:root() + for _, match in query:iter_matches(root, source) do + for id, node in pairs(match) do + if query.captures[id] == 'tagname' then + -- Only accept a *tag* when there is white space (or nothing) before it + -- and it is followed by a white character or end-of-line. + local _, _, start_byte, _, _, end_byte = node[1]:parent():range(true) + local before = source:sub(start_byte, start_byte) + local after = source:sub(end_byte + 1, end_byte + 1) + if before:match('^[ \t\n\r]?$') and after:match('^[ \t\n\r]?$') then + local tagname = ts.get_node_text(node[1], source) + local escaped = tagname:gsub('[\\/]', '\\%0') + local searchcmd = '/*' .. escaped .. '*' + table.insert(tags, { tagname, filename, searchcmd }) + end + end + end + end +end + +--- Extract tags from helpfiles and combine in a single 'tags' file. +--- @param helpfiles string[] list of helpfiles +--- @param outpath string path to write the 'tags' file to. +--- @param include_helptags_tag boolean true if the 'help-tags' tag should be included +local function gen_tagsfile(helpfiles, outpath, include_helptags_tag) + ---@type Tag[] + local tags = {} + + -- (1) extract tags from all files + for _, file in ipairs(helpfiles) do + extract_tags(tags, file) + end + + if include_helptags_tag then + table.insert(tags, { 'help-tags', 'tags', '1' }) + end + + if vim.tbl_isempty(tags) then + return + end + + -- (2) sort alphabetically on tag name + table.sort(tags, function(a, b) + return a[1] < b[1] + end) + + -- (3) check duplicates + local has_duplicates = find_duplicates(tags) + + -- (4) write tags to file + local f = assert(io.open(outpath, 'w')) + for _, tag in ipairs(tags) do + f:write(table.concat(tag, '\t') .. '\n') + end + f:close() + + -- tags file has to be written before we can error + if has_duplicates then + error('duplicate tags') + end +end + +--- Create a "tags" file for all help files in the given directory. +--- +--- The directory {dir} is generally a "doc" directory that contains "*.txt" +--- helpfiles. +--- +--- @param dir string? Path to directory with help files. If `nil` (or |vim.NIL|), +--- generate tags for every `doc` directory in the runtimepath. +--- @param include_index_tag? boolean (default: false) Whether to include the "help-tags" tag. +function M.gen_tags(dir, include_index_tag) + if dir == vim.NIL then + dir = nil + end + vim.validate('dir', dir, 'string', true) + vim.validate('include_index_tag', include_index_tag, 'boolean', true) + + local dirs = dir and { vim.fs.normalize(dir) } or vim.api.nvim_get_runtime_file('doc', true) + local vimruntime = vim.fs.normalize(vim.fs.joinpath(vim.env.VIMRUNTIME, 'doc')) + + for _, directory in ipairs(dirs) do + local files = vim.fs.find(function(name, _) + return vim.endswith(name, '.txt') + end, { path = directory, type = 'file', limit = math.huge }) + + local outpath = vim.fs.joinpath(directory, 'tags') + gen_tagsfile(files, outpath, include_index_tag or directory == vimruntime) + + -- handle translated help files per language + local translated = vim.fs.find(function(name, _) + -- "*.[a-z][a-z]x", see :help help-translated + return name:match('%.%l%lx', -4) + end, { path = directory, type = 'file', limit = math.huge }) + + -- categorize translated files per two-letter language code + ---@type table + local per_lang = {} + for _, file in ipairs(translated) do + -- extract language code "nl" from filename "plugin.nlx" + local lang = file:sub(-3, -2) + per_lang[lang] = per_lang[lang] or {} + table.insert(per_lang[lang], file) + end + + for lang, langfiles in pairs(per_lang) do + local tagsfile = vim.fs.joinpath(directory, 'tags-' .. lang) + gen_tagsfile(langfiles, tagsfile, include_index_tag or directory == vimruntime) + end + end +end + return M diff --git a/src/nvim/help.c b/src/nvim/help.c index fcf32d7263..940096f926 100644 --- a/src/nvim/help.c +++ b/src/nvim/help.c @@ -476,316 +476,24 @@ void ex_viusage(exarg_T *eap) do_cmdline_cmd("help normal-index"); } -/// Generate tags in one help directory -/// -/// @param dir Path to the doc directory -/// @param ext Suffix of the help files (".txt", ".itx", ".frx", etc.) -/// @param tagname Name of the tags file ("tags" for English, "tags-fr" for -/// French) -/// @param add_help_tags Whether to add the "help-tags" tag -/// @param ignore_writeerr ignore write error -static void helptags_one(char *dir, const char *ext, const char *tagfname, bool add_help_tags, - bool ignore_writeerr) - FUNC_ATTR_NONNULL_ALL -{ - garray_T ga; - int filecount; - char **files; - char *s; - - // Find all *.txt files. - size_t dirlen = xstrlcpy(NameBuff, dir, sizeof(NameBuff)); - if (dirlen >= MAXPATHL - || xstrlcat(NameBuff, "/**/*", sizeof(NameBuff)) >= MAXPATHL // NOLINT - || xstrlcat(NameBuff, ext, sizeof(NameBuff)) >= MAXPATHL) { - emsg(_(e_fnametoolong)); - return; - } - - // Note: We cannot just do `&NameBuff` because it is a statically sized array - // so `NameBuff == &NameBuff` according to C semantics. - char *buff_list[1] = { NameBuff }; - const int res = gen_expand_wildcards(1, buff_list, &filecount, &files, - EW_FILE|EW_SILENT); - if (res == FAIL || filecount == 0) { - if (!got_int) { - semsg(_("E151: No match: %s"), NameBuff); - } - if (res != FAIL) { - FreeWild(filecount, files); - } - return; - } - - // Open the tags file for writing. - // Do this before scanning through all the files. - memcpy(NameBuff, dir, dirlen + 1); - if (!add_pathsep(NameBuff) - || xstrlcat(NameBuff, tagfname, sizeof(NameBuff)) >= MAXPATHL) { - emsg(_(e_fnametoolong)); - return; - } - - FILE *const fd_tags = os_fopen(NameBuff, "w"); - if (fd_tags == NULL) { - if (!ignore_writeerr) { - semsg(_("E152: Cannot open %s for writing"), NameBuff); - } - FreeWild(filecount, files); - return; - } - - // If using the "++t" argument or generating tags for "$VIMRUNTIME/doc" - // add the "help-tags" tag. - ga_init(&ga, (int)sizeof(char *), 100); - if (add_help_tags - || path_equal("$VIMRUNTIME/doc", dir, kPathCmpExpand)) { - size_t s_len = 18 + strlen(tagfname); - s = xmalloc(s_len); - snprintf(s, s_len, "help-tags\t%s\t1\n", tagfname); - GA_APPEND(char *, &ga, s); - } - - // Go over all the files and extract the tags. - for (int fi = 0; fi < filecount && !got_int; fi++) { - FILE *const fd = os_fopen(files[fi], "r"); - if (fd == NULL) { - semsg(_("E153: Unable to open %s for reading"), files[fi]); - continue; - } - const char *const fname = files[fi] + dirlen + 1; - - bool in_example = false; - while (!vim_fgets(IObuff, IOSIZE, fd) && !got_int) { - if (in_example) { - // skip over example; a non-white in the first column ends it - if (vim_strchr(" \t\n\r", (uint8_t)IObuff[0])) { - continue; - } - in_example = false; - } - char *p1 = vim_strchr(IObuff, '*'); // find first '*' - while (p1 != NULL) { - char *p2 = strchr(p1 + 1, '*'); // Find second '*'. - if (p2 != NULL && p2 > p1 + 1) { // Skip "*" and "**". - for (s = p1 + 1; s < p2; s++) { - if (*s == ' ' || *s == '\t' || *s == '|') { - break; - } - } - - // Only accept a *tag* when it consists of valid - // characters, there is white space before it and is - // followed by a white character or end-of-line. - if (s == p2 - && (p1 == IObuff || p1[-1] == ' ' || p1[-1] == '\t') - && (vim_strchr(" \t\n\r", (uint8_t)s[1]) != NULL - || s[1] == NUL)) { - *p2 = NUL; - p1++; - size_t s_len = (size_t)(p2 - p1) + strlen(fname) + 2; - s = xmalloc(s_len); - GA_APPEND(char *, &ga, s); - snprintf(s, s_len, "%s\t%s", p1, fname); - - // find next '*' - p2 = vim_strchr(p2 + 1, '*'); - } - } - p1 = p2; - } - size_t off = strlen(IObuff); - if (off >= 2 && IObuff[off - 1] == '\n') { - off -= 2; - while (off > 0 && (ASCII_ISLOWER(IObuff[off]) || ascii_isdigit(IObuff[off]))) { - off--; - } - if (IObuff[off] == '>' && (off == 0 || IObuff[off - 1] == ' ')) { - in_example = true; - } - } - line_breakcheck(); - } - - fclose(fd); - } - - FreeWild(filecount, files); - - if (!got_int && ga.ga_data != NULL) { - // Sort the tags. - sort_strings(ga.ga_data, ga.ga_len); - - // Check for duplicates. - for (int i = 1; i < ga.ga_len; i++) { - char *p1 = ((char **)ga.ga_data)[i - 1]; - char *p2 = ((char **)ga.ga_data)[i]; - while (*p1 == *p2) { - if (*p2 == '\t') { - *p2 = NUL; - vim_snprintf(NameBuff, MAXPATHL, - _("E154: Duplicate tag \"%s\" in file %s/%s"), - ((char **)ga.ga_data)[i], dir, p2 + 1); - emsg(NameBuff); - *p2 = '\t'; - break; - } - p1++; - p2++; - } - } - - // Write the tags into the file. - for (int i = 0; i < ga.ga_len; i++) { - s = ((char **)ga.ga_data)[i]; - if (strncmp(s, "help-tags\t", 10) == 0) { - // help-tags entry was added in formatted form - fputs(s, fd_tags); - } else { - fprintf(fd_tags, "%s\t/" "*", s); - for (char *p1 = s; *p1 != '\t'; p1++) { - // insert backslash before '\\' and '/' - if (*p1 == '\\' || *p1 == '/') { - putc('\\', fd_tags); - } - putc(*p1, fd_tags); - } - fprintf(fd_tags, "*\n"); - } - } - } - - GA_DEEP_CLEAR_PTR(&ga); - fclose(fd_tags); // there is no check for an error... -} - -/// Generate tags in one help directory, taking care of translations. -static void do_helptags(char *dirname, bool add_help_tags, bool ignore_writeerr) - FUNC_ATTR_NONNULL_ALL -{ - garray_T ga; - char lang[2]; - char ext[5]; - char fname[8]; - int filecount; - char **files; - - // Get a list of all files in the help directory and in subdirectories. - xstrlcpy(NameBuff, dirname, sizeof(NameBuff)); - if (!add_pathsep(NameBuff) - || xstrlcat(NameBuff, "**", sizeof(NameBuff)) >= MAXPATHL) { - emsg(_(e_fnametoolong)); - return; - } - - // Note: We cannot just do `&NameBuff` because it is a statically sized array - // so `NameBuff == &NameBuff` according to C semantics. - char *buff_list[1] = { NameBuff }; - if (gen_expand_wildcards(1, buff_list, &filecount, &files, - EW_FILE|EW_SILENT) == FAIL - || filecount == 0) { - semsg(_("E151: No match: %s"), NameBuff); - return; - } - - // Go over all files in the directory to find out what languages are - // present. - int j; - ga_init(&ga, 1, 10); - for (int i = 0; i < filecount; i++) { - int len = (int)strlen(files[i]); - if (len <= 4) { - continue; - } - - if (STRICMP(files[i] + len - 4, ".txt") == 0) { - // ".txt" -> language "en" - lang[0] = 'e'; - lang[1] = 'n'; - } else if (files[i][len - 4] == '.' - && ASCII_ISALPHA(files[i][len - 3]) - && ASCII_ISALPHA(files[i][len - 2]) - && TOLOWER_ASC(files[i][len - 1]) == 'x') { - // ".abx" -> language "ab" - lang[0] = (char)TOLOWER_ASC(files[i][len - 3]); - lang[1] = (char)TOLOWER_ASC(files[i][len - 2]); - } else { - continue; - } - - // Did we find this language already? - for (j = 0; j < ga.ga_len; j += 2) { - if (strncmp(lang, ((char *)ga.ga_data) + j, 2) == 0) { - break; - } - } - if (j == ga.ga_len) { - // New language, add it. - ga_grow(&ga, 2); - ((char *)ga.ga_data)[ga.ga_len++] = lang[0]; - ((char *)ga.ga_data)[ga.ga_len++] = lang[1]; - } - } - - // Loop over the found languages to generate a tags file for each one. - for (j = 0; j < ga.ga_len; j += 2) { - STRCPY(fname, "tags-xx"); - fname[5] = ((char *)ga.ga_data)[j]; - fname[6] = ((char *)ga.ga_data)[j + 1]; - if (fname[5] == 'e' && fname[6] == 'n') { - // English is an exception: use ".txt" and "tags". - fname[4] = NUL; - STRCPY(ext, ".txt"); - } else { - // Language "ab" uses ".abx" and "tags-ab". - STRCPY(ext, ".xxx"); - ext[1] = fname[5]; - ext[2] = fname[6]; - } - helptags_one(dirname, ext, fname, add_help_tags, ignore_writeerr); - } - - ga_clear(&ga); - FreeWild(filecount, files); -} - -static bool helptags_cb(int num_fnames, char **fnames, bool all, void *cookie) - FUNC_ATTR_NONNULL_ALL -{ - for (int i = 0; i < num_fnames; i++) { - do_helptags(fnames[i], *(bool *)cookie, true); - if (!all) { - return true; - } - } - - return num_fnames > 0; -} - /// ":helptags" void ex_helptags(exarg_T *eap) { - expand_T xpc; bool add_help_tags = false; - // Check for ":helptags ++t {dir}". + // Check for ++t in ":helptags ++t {dir}". if (strncmp(eap->arg, "++t", 3) == 0 && ascii_iswhite(eap->arg[3])) { add_help_tags = true; eap->arg = skipwhite(eap->arg + 3); } + typval_T tv_args[] = { + { .v_type = VAR_STRING, .vval.v_string = eap->arg }, + { .v_type = VAR_BOOL, .vval.v_bool = add_help_tags ? kBoolVarTrue : kBoolVarFalse }, + { .v_type = VAR_UNKNOWN }, + }; if (strcmp(eap->arg, "ALL") == 0) { - do_in_path(p_rtp, "", "doc", DIP_ALL + DIP_DIR, helptags_cb, &add_help_tags); - } else { - ExpandInit(&xpc); - xpc.xp_context = EXPAND_DIRECTORIES; - char *dirname = - ExpandOne(&xpc, eap->arg, NULL, WILD_LIST_NOTFOUND|WILD_SILENT, WILD_EXPAND_FREE); - if (dirname == NULL || !os_isdir(dirname)) { - semsg(_("E150: Not a directory: %s"), eap->arg); - } else { - do_helptags(dirname, add_help_tags, false); - } - xfree(dirname); + tv_args[0] = (typval_T){ .v_type = VAR_SPECIAL, .vval.v_special = kSpecialVarNull }; } + nlua_call_typval("vim._core.help", "gen_tags", tv_args, NULL); } diff --git a/test/functional/ex_cmds/help_spec.lua b/test/functional/ex_cmds/help_spec.lua index dcba0ac872..1a7df82df3 100644 --- a/test/functional/ex_cmds/help_spec.lua +++ b/test/functional/ex_cmds/help_spec.lua @@ -1,11 +1,12 @@ local t = require('test.testutil') local n = require('test.functional.testnvim')() -local describe, it, before_each, finally = t.describe, t.it, t.before_each, t.finally +local describe, it, before_each, after_each, finally = + t.describe, t.it, t.before_each, t.after_each, t.finally local clear = n.clear +local eval = n.eval local command = n.command local eq = t.eq -local pcall_err = t.pcall_err local fn = n.fn local api = n.api local mkdir = t.mkdir @@ -324,3 +325,83 @@ describe(':help', function() eq('*…*', api.nvim_get_current_line()) end) end) + +describe(':helptags', function() + before_each(function() + for _, sfx in ipairs({ '', '2' }) do + fn.mkdir(('Xhelptags%s/doc'):format(sfx), 'p') + for _, tag in ipairs({ 'Xa', 'Xb' }) do + write_file(('Xhelptags%s/doc/%s%s.txt'):format(sfx, tag, sfx), ('*%s%s*'):format(tag, sfx)) + end + end + + clear() + command('set rtp+=Xhelptags,Xhelptags2') + end) + + after_each(function() + rmdir('Xhelptags') + rmdir('Xhelptags2') + end) + + it('requires an argument', function() + local msg = t.pcall_err(command, 'helptags') + eq(true, msg:find('E471') ~= nil) + end) + + it('{dir}', function() + command('helptags Xhelptags/doc') + + eq(eval("['Xa Xa.txt /*Xa*','Xb Xb.txt /*Xb*']"), eval("readfile('Xhelptags/doc/tags')")) + + command('help Xa') + eq('*Xa*', api.nvim_get_current_line()) + end) + + it('ALL', function() + command('helptags ALL') + + eq(eval("['Xa Xa.txt /*Xa*','Xb Xb.txt /*Xb*']"), eval("readfile('Xhelptags/doc/tags')")) + eq(eval("['Xa2 Xa2.txt /*Xa2*','Xb2 Xb2.txt /*Xb2*']"), eval("readfile('Xhelptags2/doc/tags')")) + + command('help Xa2') + eq('*Xa2*', api.nvim_get_current_line()) + end) + + it('++t', function() + command('helptags ++t Xhelptags/doc') + eq('help-tags tags 1', eval("readfile('Xhelptags/doc/tags')[-1]")) + end) + + it('generates help-tag tag for VIMRUNTIME', function() + command('let $VIMRUNTIME="Xhelptags"') + command('helptags Xhelptags/doc') + eq('help-tags tags 1', eval("readfile('Xhelptags/doc/tags')[-1]")) + end) + + it('errors on duplicate tags', function() + -- duplicate tags in different files + write_file('Xhelptags/doc/Xd.txt', '*Xa*', nil, true) + local msg = t.pcall_err(command, 'helptags Xhelptags/doc') + eq(true, msg:find('E154') ~= nil) + + -- tags file should still be generated + eq(1, eval("filereadable('Xhelptags/doc/tags')")) + + os.remove('Xhelptags/doc/Xd.txt') + + -- duplicate tags in same file + write_file('Xhelptags/doc/Xa.txt', '\n*Xa*', nil, true) + + msg = t.pcall_err(command, 'helptags Xhelptags/doc') + eq(true, msg:find('E154') ~= nil) + + eq(1, eval("filereadable('Xhelptags/doc/tags')")) + end) + + it('with translated help files', function() + write_file('Xhelptags/doc/Xa.nlx', '*Xa*', nil, true) + command('helptags Xhelptags/doc') + eq(1, eval("filereadable('Xhelptags/doc/tags-nl')")) + end) +end)