From d663d8f4b3116a9cad4ade936218ccb4ed723c30 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Wed, 9 Sep 2026 18:43:20 +0100 Subject: [PATCH] perf(help): generate helptags with LPeg Problem: Generating helptags builds and queries a full vimdoc syntax tree. This is much slower than the previous scanner and also happens before discovering that the tags file cannot be written. Solution: Extract tags with an LPeg grammar, reading each file once and skipping example blocks and ordinary text. Open the output first so unwritable directories can return immediately. This restores legacy handling of tags inside inline code spans. On an arm64 Mac, the 5.36 MB runtime corpus takes 24 ms instead of 604 ms, with identical generated tags. AI-assisted --- runtime/lua/vim/_core/help.lua | 107 +++++++++++++------------- test/functional/ex_cmds/help_spec.lua | 53 +++++++++++-- 2 files changed, 99 insertions(+), 61 deletions(-) diff --git a/runtime/lua/vim/_core/help.lua b/runtime/lua/vim/_core/help.lua index b957621244..58acb9c5b2 100644 --- a/runtime/lua/vim/_core/help.lua +++ b/runtime/lua/vim/_core/help.lua @@ -417,61 +417,64 @@ end ---Report duplicate tags (as errmsg, not exception-throwing error). ---@param tags string[] sorted tags file lines local function report_duplicates(tags) - local prevtag, prevfn = '', '' + local prevtag - for _, tagline in ipairs(tags) do - local curtag, curfn = tagline:match('^([^\t]*)\t([^\t]*)') + for i, tagline in ipairs(tags) do + local curtag = tagline:match('^[^\t]*') if curtag == prevtag then + local curfn = tagline:match('\t([^\t]*)') + local prevfn = tags[i - 1]:match('\t([^\t]*)') local filenames = prevfn ~= curfn and (curfn .. ' and ' .. prevfn) or curfn echo_err(('E154: Duplicate tag "%s" in %s'):format(curtag, filenames)) end prevtag = curtag - prevfn = curfn end end +local helptags_pattern ---@type vim.lpeg.Pattern +do + local P, S, R, B, C, Ct = vim.lpeg.P, vim.lpeg.S, vim.lpeg.R, vim.lpeg.B, vim.lpeg.C, vim.lpeg.Ct + local any = P(1) + local eof = -any + local space = S(' \t') + -- Accept LF, CRLF, and an unterminated final line. + local newline = P('\r') ^ -1 * P('\n') + local line_end = newline + P('\r') ^ -1 * eof + local line_start = -B(any) + B(P('\n')) + -- Assert surrounding whitespace without consuming it, so adjacent tags can share it. + local tag = (line_start + B(space)) + * P('*') + * C((any - S('* \t|\n')) ^ 1) + * P('*') + * #(space + line_end) + -- An example continues through blank or indented lines. + local example_line = newline + space * (any - P('\n')) ^ 0 * (P('\n') + eof) + local example = (line_start + B(P(' '))) + * P('>') + * R('az', '09') ^ 0 + * line_end + * example_line ^ 0 + -- Skip ordinary text in spans; consume invalid markers one character at a time. + local text = (any - S('*>')) ^ 1 + any + helptags_pattern = Ct((example + tag + text) ^ 0) +end + ---Extract tags from {file} and add to list of tags. Modifies {tags}. ---@param tags string[] ---@param file string ---@param name string Path of {file} relative to the help directory, as stored in the tags file. local function extract_tags(tags, file, name) - local ts = vim.treesitter - local ok, source = pcall(vim.fn.readblob, file) - if not ok then + local f = io.open(file, 'r') + if not f then echo_err(('E153: Unable to open %s for reading'):format(file)) return end - --- @cast source string - -- The grammar treats "\r" as part of a word, so CRLF files would yield bogus tags. - source = source:gsub('\r\n', '\n') + -- Read once to avoid stdio calls and locking for every line. + local source = assert(f:read('*a')) + f:close() - local query = ts.query.parse('vimdoc', '(tag (word) @tagname)') - local parser = ts.get_string_parser(source, 'vimdoc') - - local tree = assert(parser:parse()) - local root = tree[1]:root() - for _, match in query:iter_matches(root, source) do - for id, node in pairs(match) do - if query.captures[id] == 'tagname' then - -- Only accept a *tag* when it is closed, has no "|" (which would break |links|), there is - -- whitespace (or nothing) before it, and followed by whitespace or end-of-line. - local tag_node = assert(node[1]:parent()) - local _, _, start_byte = tag_node:start() - local _, _, end_byte = tag_node:end_() - local before = source:sub(start_byte, start_byte) - local after = source:sub(end_byte + 1, end_byte + 1) - local tagname = ts.get_node_text(node[1], source) - if - source:sub(end_byte, end_byte) == '*' - and not tagname:find('|', 1, true) - and before:match('^[ \t\n\r]?$') - and after:match('^[ \t\n\r]?$') - then - local escaped = tagname:gsub('[\\/]', '\\%0') - table.insert(tags, ('%s\t%s\t/*%s*'):format(tagname, name, escaped)) - end - end - end + for _, tagname in ipairs(helptags_pattern:match(source)) do + tags[#tags + 1] = ('%s\t%s\t/*%s*'):format(tagname, name, tagname:gsub('[\\/]', '\\%0')) end end @@ -482,6 +485,15 @@ end --- @param include_helptags_tag boolean true if the 'help-tags' tag should be included --- @param ignore_writeerr boolean don't report a tags file that cannot be written local function gen_tagsfile(helpfiles, dir, outpath, include_helptags_tag, ignore_writeerr) + -- Avoid scanning helpfiles when the output cannot be written (:helptags ALL). + local f = io.open(outpath, 'w') + if not f then + if not ignore_writeerr then + echo_err(('E152: Cannot open %s for writing'):format(outpath)) + end + return + end + ---@type string[] Tags file lines: "tagfilesearch command". local tags = {} @@ -502,13 +514,6 @@ local function gen_tagsfile(helpfiles, dir, outpath, include_helptags_tag, ignor report_duplicates(tags) -- (4) write tags to file - local f = io.open(outpath, 'w') - if not f then - if not ignore_writeerr then - echo_err(('E152: Cannot open %s for writing'):format(outpath)) - end - return - end for _, tag in ipairs(tags) do f:write(tag, '\n') end @@ -530,13 +535,6 @@ function M.gen_tags(dir, include_index_tag) vim.validate('dir', dir, 'string', true) vim.validate('include_index_tag', include_index_tag, 'boolean', true) - if not pcall(function() - vim.treesitter.language.add('vimdoc') - end) then - echo_err('Cannot generate helptags: no "vimdoc" parser') - return - end - local dirs = dir and { vim.fs.normalize(dir) } or vim.api.nvim_get_runtime_file('doc', true) local vimruntime = vim.fs.normalize(vim.fs.joinpath(vim.env.VIMRUNTIME, 'doc')) @@ -546,9 +544,12 @@ function M.gen_tags(dir, include_index_tag) end for _, directory in ipairs(dirs) do + -- Resolve once for traversal and relpath(), avoiding getcwd() for every file. + -- Keep directory for messages and the VIMRUNTIME comparison. + local absdir = vim.fs.abspath(directory) local files = vim.fs.find(function(name, _) return helpfile_lang(name) ~= nil - end, { path = directory, type = 'file', limit = math.huge }) + end, { path = absdir, type = 'file', limit = math.huge }) if vim.tbl_isempty(files) then echo_err(('E151: No match: %s'):format(vim.fs.joinpath(directory, '**/*.txt'))) @@ -571,7 +572,7 @@ function M.gen_tags(dir, include_index_tag) local ignore_writeerr = dir == nil gen_tagsfile( langfiles, - directory, + absdir, outpath, include_index_tag or directory == vimruntime, ignore_writeerr diff --git a/test/functional/ex_cmds/help_spec.lua b/test/functional/ex_cmds/help_spec.lua index f1cb7f0090..23cc11d1a4 100644 --- a/test/functional/ex_cmds/help_spec.lua +++ b/test/functional/ex_cmds/help_spec.lua @@ -387,12 +387,19 @@ describe(':helptags', function() -- CRLF helpfile: "\r" must not confuse the parser into finding tags in an example. write_file('Xhelptags/doc/Xe.txt', '*Xe*\r\n>\r\n\t+-----+\r\n\t|/* a.c */ |/* b.c */ |\r\n') - command('helptags Xhelptags/doc') - - eq( - eval("['Xa Xa.txt /*Xa*','Xb Xb.txt /*Xb*','Xc sub/Xc.txt /*Xc*','Xe Xe.txt /*Xe*']"), - eval("readfile('Xhelptags/doc/tags')") - ) + for _, dir in ipairs({ + 'Xhelptags/doc', + './Xhelptags/doc', + fn.fnamemodify('Xhelptags/doc', ':p'), + }) do + command('helptags ' .. fn.fnameescape(dir)) + eq({ + 'Xa\tXa.txt\t/*Xa*', + 'Xb\tXb.txt\t/*Xb*', + 'Xc\tsub/Xc.txt\t/*Xc*', + 'Xe\tXe.txt\t/*Xe*', + }, fn.readfile('Xhelptags/doc/tags')) + end command('help Xa') eq('*Xa*', api.nvim_get_current_line()) @@ -401,6 +408,36 @@ describe(':helptags', function() eq('*Xc*', api.nvim_get_current_line()) end) + it('ignores tags in examples and resumes after unindented text', function() + for _, newline in ipairs({ '\n', '\r\n' }) do + write_file( + 'Xhelptags/doc/Xa.txt', + table.concat({ + '*Xa* *Xz*', + 'Some ordinary prose.', + 'prefix*invalid* *invalid*suffix *bad|tag* **', + '>lua', + '', + ' *example*', + 'End of example.', + '', + ' *Xc*', + 'More ordinary prose.', + '*Xd*', -- No final newline. + }, newline), + true + ) + command('helptags Xhelptags/doc') + eq({ + 'Xa\tXa.txt\t/*Xa*', + 'Xb\tXb.txt\t/*Xb*', + 'Xc\tXa.txt\t/*Xc*', + 'Xd\tXa.txt\t/*Xd*', + 'Xz\tXa.txt\t/*Xz*', + }, fn.readfile('Xhelptags/doc/tags')) + end + end) + it('overwrites existing tags file', function() write_file('Xhelptags/doc/tags', 'Xstale Xold.txt /*Xstale*', nil, true) write_file('Xhelptags/doc/Xa.txt', 'no tags here', nil, true) @@ -450,7 +487,7 @@ describe(':helptags', function() -- duplicate tags in different files write_file('Xhelptags/doc/Xd.txt', '*Xa*', nil, true) local msg = t.pcall_err(command, 'helptags Xhelptags/doc') - eq(true, msg:find('E154') ~= nil) + eq(true, msg:find('E154: Duplicate tag "Xa" in Xd.txt and Xa.txt', 1, true) ~= nil) -- tags file should still be generated eq(1, eval("filereadable('Xhelptags/doc/tags')")) @@ -461,7 +498,7 @@ describe(':helptags', function() write_file('Xhelptags/doc/Xa.txt', '\n*Xa*', nil, true) msg = t.pcall_err(command, 'helptags Xhelptags/doc') - eq(true, msg:find('E154') ~= nil) + eq(true, msg:find('E154: Duplicate tag "Xa" in Xa.txt', 1, true) ~= nil) eq(1, eval("filereadable('Xhelptags/doc/tags')"))