refactor(help): reuse help.lua to generate build tags

Make gen_helptags.lua call help.lua instead of keeping its own tag
parser. Builds and :helptags now use the same code.

Allow the shared code to run without Neovim, and fail the build when tag
generation reports an error.

AI-assisted

runtime/lua/vim/_core/help.lua

@@ -1,6 +1,7 @@
 local M = {}
 
 local echo_err = require('vim._core.util').echo_err
+local fs = require('vim.fs')
 
 local tag_exceptions = {
   -- Interpret asterisk (star, '*') literal but name it 'star'
@@ -414,7 +415,7 @@ local function helpfile_lang(file)
   return ext == '.txt' and 'en' or ext:match('^%.(%a%a)x$')
 end
 
----Report duplicate tags (as errmsg, not exception-throwing error).
+---Report duplicate tags.
 ---@param tags string[] sorted tags file lines
 local function report_duplicates(tags)
   local prevtag
@@ -482,9 +483,9 @@ end
 --- @param helpfiles string[] list of helpfiles
 --- @param dir string Help directory; tag entries name the helpfiles relative to it.
 --- @param outpath string path to write the 'tags' file to.
---- @param include_helptags_tag boolean true if the 'help-tags' tag should be included
+--- @param index_tag string? Filename for the "help-tags" entry, if included
 --- @param ignore_writeerr boolean don't report a tags file that cannot be written
-local function gen_tagsfile(helpfiles, dir, outpath, include_helptags_tag, ignore_writeerr)
+function M.gen_tagsfile(helpfiles, dir, outpath, index_tag, ignore_writeerr)
   -- Avoid scanning helpfiles when the output cannot be written (:helptags ALL).
   local f = io.open(outpath, 'w')
   if not f then
@@ -499,18 +500,21 @@ local function gen_tagsfile(helpfiles, dir, outpath, include_helptags_tag, ignor
 
   -- (1) extract tags from all files
   for _, file in ipairs(helpfiles) do
-    extract_tags(tags, file, vim.fs.relpath(dir, file) or vim.fs.basename(file))
+    extract_tags(tags, file, fs.relpath(dir, file) or fs.basename(file))
   end
 
-  if include_helptags_tag then
-    table.insert(tags, ('help-tags\t%s\t1'):format(vim.fs.basename(outpath)))
+  if index_tag then
+    table.insert(tags, ('help-tags\t%s\t1'):format(index_tag))
   end
 
   -- (2) sort by byte value, as |tags-file-format| requires.
-  -- Note: vim.fn.sort() compares bytes, PUC Lua "<" compares with strcoll().
-  tags = vim.fn.sort(tags)
+  -- PUC Lua uses strcoll(), so use C collation for this sort.
+  local locale = os.setlocale(nil, 'collate')
+  os.setlocale('C', 'collate')
+  table.sort(tags)
+  os.setlocale(locale, 'collate')
 
-  -- (3) report duplicates (non-fatal errmsg: the tags file is still written)
+  -- (3) report duplicates
   report_duplicates(tags)
 
   -- (4) write tags to file
@@ -570,11 +574,11 @@ function M.gen_tags(dir, include_index_tag)
       local outpath = vim.fs.joinpath(directory, tagsfile)
       -- ":helptags ALL" walks 'runtimepath', which may contain read-only directories.
       local ignore_writeerr = dir == nil
-      gen_tagsfile(
+      M.gen_tagsfile(
         langfiles,
         absdir,
         outpath,
-        include_index_tag or directory == vimruntime,
+        (include_index_tag or directory == vimruntime) and tagsfile or nil,
         ignore_writeerr
       )
     end

runtime/lua/vim/_core/util.lua

@@ -197,9 +197,12 @@ function M.cmd_errmsg(err)
   return (err:gsub('^Lua:%s*', ''))
 end
 
---- Utility function for displaying vim error codes (EXX)
+--- Display a Vim error code (EXX), or raise an error without editor APIs.
 --- @param msg string
 function M.echo_err(msg)
+  if not vim.api then
+    error(msg, 2)
+  end
   vim.api.nvim_echo({ { msg } }, true, { err = true })
 end
 

runtime/lua/vim/fs.lua

@@ -44,8 +44,8 @@ local uv = vim.uv
 
 local M = {}
 
-local iswin = vim.fn.has('win32') == 1
-local os_sep = iswin and '\\' or '/'
+local os_sep = package.config:sub(1, 1)
+local iswin = os_sep == '\\'
 
 --- Iterate over all the parents of the given path (not expanded/resolved, the caller must do that).
 ---

src/gen/gen_helptags.lua

@@ -1,87 +1,29 @@
----@diagnostic disable: no-unknown
--- Does the same as `nvim -c "helptags [++t] doc" -c quit`
--- without needing to run a "nvim" binary, which is needed for cross-compiling.
---
+-- Generate build tags using the runtime scanner, including with nlua0.
 -- Usage: nlua0 gen_helptags.lua {out} {dir} [++t]
--- The tags file is sorted by byte value, but PUC Lua "<" compares with strcoll().
-os.setlocale('C', 'collate')
+local scriptdir = arg[0]:match('^(.*[/\\])') or './'
+local luadir = scriptdir .. '../../runtime/lua/'
+package.path = luadir .. '?.lua;' .. package.path
 
-local out = arg[1]
-local dir = arg[2]
-local add_help_tags = arg[3] == '++t'
+local fs = require('vim.fs')
+-- Load the source version: NVIM_HOST_PRG may embed an older help module.
+local help = dofile(luadir .. 'vim/_core/help.lua')
 
-local dirfd = assert(vim.uv.fs_opendir(dir, nil, 1))
+local dir = fs.abspath(arg[2])
 local files = {}
+local scan = assert(vim.uv.fs_scandir(dir))
 while true do
-  local file = dirfd:readdir()
-  if file == nil then
+  local name, kind = vim.uv.fs_scandir_next(scan)
+  if not name then
     break
   end
-  if file[1].type == 'file' and vim.endswith(file[1].name, '.txt') then
-    table.insert(files, file[1].name)
+  if kind == 'file' and name:sub(-4) == '.txt' then
+    files[#files + 1] = fs.joinpath(dir, name)
   end
 end
 
-local tags = {}
-for _, fn in ipairs(files) do
-  local in_example = false
-  for line in io.lines(dir .. '/' .. fn) do
-    if in_example then
-      local first = string.sub(line, 1, 1)
-      if first ~= ' ' and first ~= '\t' and first ~= '' then
-        in_example = false
-      end
-    end
-    local chunks = vim.split(line, '*', { plain = true })
-    local next_valid = false
-    local n_chunks = #chunks
-    for i, chunk in ipairs(chunks) do
-      if next_valid and not in_example then
-        if #chunk > 0 and string.find(chunk, '[ \t|]') == nil then
-          local next = string.sub(chunks[i + 1], 1, 1)
-          if next == ' ' or next == '\t' or (i == n_chunks - 1 and next == '') then
-            table.insert(tags, { chunk, fn })
-          end
-        end
-      end
+help.gen_tagsfile(files, dir, arg[1], arg[3] == '++t' and 'tags' or nil, false)
 
-      if i == n_chunks - 1 then
-        break
-      end
-      next_valid = false
-      local lastend = string.sub(chunk, -1) -- "" for empty string
-      if lastend == ' ' or lastend == '\t' or (i == 1 and lastend == '') then
-        next_valid = true
-      end
-    end
-
-    if line:find('^>[a-z0-9]*$') or line:find(' >[a-z0-9]*$') then
-      in_example = true
-    end
-  end
+-- nvim -l exits successfully after echo_err(), so fail the build explicitly.
+if vim.v and vim.v.errmsg ~= '' then
+  os.exit(1)
 end
-
-if add_help_tags then
-  table.insert(tags, { 'help-tags', 'tags' })
-end
-table.sort(tags, function(a, b)
-  return a[1] < b[1]
-end)
-
-local f = assert(io.open(out, 'w'))
-local lasttagname, lastfn = nil, nil
-for _, tag in ipairs(tags) do
-  local tagname, fn = unpack(tag)
-  if tagname == lasttagname then
-    error('duplicate tags in ' .. fn .. (lastfn ~= fn and (' and ' .. lastfn) or ''))
-  end
-  lasttagname, lastfn = tagname, fn
-
-  if tagname == 'help-tags' then
-    f:write(tagname .. '\t' .. fn .. '\t1\n')
-  else
-    local escaped = string.gsub(tagname, '[\\/]', '\\%0')
-    f:write(tagname .. '\t' .. fn .. '\t/*' .. escaped .. '*\n')
-  end
-end
-f:close()