feat(docs): update parser, HTML gen

Note: although the tolerance in help_spec.lua increased, the actual
error count with the new parser decreased by about 20%. The difference
is that the old ignore_parse_error() ignored many more errors with the
old parser.

fix https://github.com/neovim/tree-sitter-vimdoc/issues/37
fix https://github.com/neovim/tree-sitter-vimdoc/issues/44
fix https://github.com/neovim/tree-sitter-vimdoc/issues/47

CONTRIBUTING.md

@@ -12,6 +12,9 @@ low-risk/isolated tasks:
   [Coverity](#coverity).
 - [Improve documentation](#documenting)
 - [Merge a Vim patch] (requires strong familiarity with Vim)
+  - NOTE: read the above link before sending improvements to "runtime files" (anything in `runtime/`).
+    - Vimscript and documentation files are (mostly) maintained by [Vim](https://github.com/vim/vim), not Nvim.
+    - Lua files are maintained by Nvim.
 
 Reporting problems
 ------------------

runtime/doc/arabic.txt

@@ -14,8 +14,9 @@ It is best to view this file with these settings within VIM's GUI: >
 	:set arabicshape
 
 
+------------------------------------------------------------------------------
 Introduction
-------------
+
 Arabic is a rather demanding language in which a number of special
 features are required.	Characters are right-to-left oriented and
 ought to appear as such on the screen (i.e. from right to left).
@@ -34,8 +35,9 @@ The commands, prompts and help files are not in Arabic, therefore
 the user interface remains the standard Vi interface.
 
 
+------------------------------------------------------------------------------
 Highlights
-----------
+
 o  Editing left-to-right files as in the original Vim hasn't changed.
 
 o  Viewing and editing files in right-to-left windows.	 File
@@ -64,8 +66,8 @@ o  Proper Bidirectional functionality is possible given Vim is
    started within a Bidi capable terminal emulator.
 
 
+------------------------------------------------------------------------------
 Arabic Fonts						*arabicfonts*
-------------
 
 Vim requires monospaced fonts of which there are many out there.
 Arabic requires ISO-8859-6 as well as Presentation Form-B fonts
@@ -75,8 +77,8 @@ Do an Internet search or check www.arabeyes.org for further
 info on where to obtain the necessary Arabic fonts.
 
 
+------------------------------------------------------------------------------
 Font Installation
------------------
 
 o  Installation of fonts for X Window systems (Unix/Linux)
 
@@ -88,8 +90,9 @@ o  Installation of fonts for X Window systems (Unix/Linux)
      %	xset +fp path_name_of_arabic_fonts_directory
 
 
+------------------------------------------------------------------------------
 Usage
------
+
 Prior to the actual usage of Arabic within Vim, a number of settings
 need to be accounted for and invoked.
 
@@ -259,8 +262,8 @@ o  Enable Arabic settings [short-cut]
       ':set arabicshape' to your vimrc file.
 
 
+------------------------------------------------------------------------------
 Keymap/Keyboard						*arabickeymap*
----------------
 
 The character/letter encoding used in Vim is the standard UTF-8.
 It is widely discouraged that any other encoding be used or even
@@ -276,7 +279,7 @@ o  Keyboard
    +  CTRL-^ in insert/replace mode toggles between Arabic/Latin mode
 
    +  Keyboard mapping is based on the Microsoft's Arabic keymap (the
-      de facto standard in the Arab world):
+      de facto standard in the Arab world): >
 
   +---------------------------------------------------------------------+
   |!   |@   |#   |$   |%   |^   |&   |*   |(   |)   |_   |+   ||   |~  ّ |
@@ -291,17 +294,18 @@ o  Keyboard
 	   |Z ~ |X  ْ |C { |V } |B لآ |N آ |M ' |< , |> . |? ؟ |
 	   |z ئ |x ء |c ؤ |v ر |b لا |n ى |m ة |, و |. ز |/ ظ |
 	   +-------------------------------------------------+
+<
 
+------------------------------------------------------------------------------
 Restrictions
-------------
 
 o  Vim in its GUI form does not currently support Bi-directionality
    (i.e. the ability to see both Arabic and Latin intermixed within
    the same line).
 
 
+------------------------------------------------------------------------------
 Known Bugs
-----------
 
 There is one known minor bug,
 

runtime/doc/builtin.txt

@@ -6790,18 +6790,24 @@ serverstart([{address}])				*serverstart()*
 		|RPC| messages. Clients can send |API| commands to the
 		returned address to control Nvim.
 
-		Returns the address string (may differ from the requested
+		Returns the address string (which may differ from the
-		{address}).
+		{address} argument, see below).
-		 
-		- If {address} contains a colon ":" it is interpreted as
-		  a TCP/IPv4/IPv6 address where the last ":" separates host
-		  and port (empty or zero assigns a random port).
-		- Else it is interpreted as a named pipe or Unix domain socket
-		  path. If there are no slashes it is treated as a name and
-		  appended to a generated path.
-		- If {address} is empty it generates a path.
 
-		Example named pipe: >
+		- If {address} has a colon (":") it is a TCP/IPv4/IPv6 address
+		  where the last ":" separates host and port (empty or zero
+		  assigns a random port).
+		- Else {address} is the path to a named pipe (except on Windows).
+		  - If {address} has no slashes ("/") it is treated as the
+		    "name" part of a generated path in this format: >
+			stdpath("run").."/{name}.{pid}.{counter}"
+<		  - If {address} is omitted the name is "nvim". >
+			:echo serverstart()
+			=> /tmp/nvim.bram/oknANW/nvim.15430.5
+
+<		Example bash command to list all Nvim servers: >
+			ls ${XDG_RUNTIME_DIR:-${TMPDIR}nvim.${USER}}/*/nvim.*.0
+
+<		Example named pipe: >
 			if has('win32')
 			  echo serverstart('\\.\pipe\nvim-pipe-1234')
 			else

runtime/doc/eval.txt

@@ -2141,9 +2141,11 @@ v:scrollstart	String describing the script or function that caused the
 		hit-enter prompt.
 
 					*v:servername* *servername-variable*
-v:servername	Primary listen-address of the current Nvim instance, the first
+v:servername	Primary listen-address of Nvim, the first item returned by
-		item returned by |serverlist()|.  Can be set by |--listen| or
+		|serverlist()|. Usually this is the named pipe created by Nvim
-		|$NVIM_LISTEN_ADDRESS| (deprecated) at startup.
+		at |startup| or given by |--listen| (or the deprecated
+		|$NVIM_LISTEN_ADDRESS| env var).
+
 		See also |serverstart()| |serverstop()|.
 		Read-only.
 

runtime/doc/motion.txt

@@ -309,7 +309,7 @@ g<Down>			[count] display lines downward.  |exclusive| motion.
 			an operator, because it's not linewise.
 
 							*-*
--  <minus>		[count] lines upward, on the first non-blank
+`-`  <minus>		[count] lines upward, on the first non-blank
 			character |linewise|.
 
 +		or					*+*

runtime/doc/starting.txt

@@ -401,18 +401,20 @@ accordingly, proceeding as follows:
 	The |-V| argument can be used to display or log what happens next,
 	useful for debugging the initializations.
 
-3. Wait for UI to connect.
+3. Start a server (unless |--listen| was given) and set |v:servername|.
+
+4. Wait for UI to connect.
 	Nvim started with |--embed| waits for the UI to connect before
 	proceeding to load user configuration.
 
-4. Setup |default-mappings| and |default-autocmds|.  Create |popup-menu|.
+5. Setup |default-mappings| and |default-autocmds|.  Create |popup-menu|.
 
-5. Enable filetype and indent plugins.
+6. Enable filetype and indent plugins.
 	This does the same as the command: >
 		:runtime! ftplugin.vim indent.vim
 <	Skipped if the "-u NONE" command line argument was given.
 
-6. Load user config (execute Ex commands from files, environment, …).
+7. Load user config (execute Ex commands from files, environment, …).
 	$VIMINIT environment variable is read as one Ex command line (separate
 	multiple commands with '|' or <NL>).
 					*config* *init.vim* *init.lua* *vimrc* *exrc*
@@ -456,19 +458,19 @@ accordingly, proceeding as follows:
 	-  The file ".nvimrc"
 	-  The file ".exrc"
 
-7. Enable filetype detection.
+8. Enable filetype detection.
 	This does the same as the command: >
 		:runtime! filetype.lua
 <	Skipped if ":filetype off" was called or if the "-u NONE" command line
 	argument was given.
 
-8. Enable syntax highlighting.
+9. Enable syntax highlighting.
 	This does the same as the command: >
 		:runtime! syntax/syntax.vim
 <	Skipped if ":syntax off" was called or if the "-u NONE" command
 	line argument was given.
 
-9. Load the plugin scripts.					*load-plugins*
+10. Load the plugin scripts.					*load-plugins*
 	This does the same as the command: >
 		:runtime! plugin/**/*.vim
 		:runtime! plugin/**/*.lua
@@ -498,21 +500,21 @@ accordingly, proceeding as follows:
 	if packages have been found, but that should not add a directory
 	ending in "after".
 
-10. Set 'shellpipe' and 'shellredir'
+11. Set 'shellpipe' and 'shellredir'
 	The 'shellpipe' and 'shellredir' options are set according to the
 	value of the 'shell' option, unless they have been set before.
 	This means that Nvim will figure out the values of 'shellpipe' and
 	'shellredir' for you, unless you have set them yourself.
 
-11. Set 'updatecount' to zero, if "-n" command argument used
+12. Set 'updatecount' to zero, if "-n" command argument used
 
-12. Set binary options if the |-b| flag was given.
+13. Set binary options if the |-b| flag was given.
 
-13. Read the |shada-file|.
+14. Read the |shada-file|.
 
-14. Read the quickfix file if the |-q| flag was given, or exit on failure.
+15. Read the quickfix file if the |-q| flag was given, or exit on failure.
 
-15. Open all windows
+16. Open all windows
 	When the |-o| flag was given, windows will be opened (but not
 	displayed yet).
 	When the |-p| flag was given, tab pages will be created (but not
@@ -522,7 +524,7 @@ accordingly, proceeding as follows:
 	Buffers for all windows will be loaded, without triggering |BufAdd|
 	autocommands.
 
-16. Execute startup commands
+17. Execute startup commands
 	If a |-t| flag was given, the tag is jumped to.
 	Commands given with |-c| and |+cmd| are executed.
 	The starting flag is reset, has("vim_starting") will now return zero.

scripts/gen_help_html.lua

@@ -74,6 +74,23 @@ local exclude_invalid = {
   ["vim.treesitter.start()"] = "treesitter.txt",
 }
 
+-- False-positive "invalid URLs".
+local exclude_invalid_urls = {
+  ["http://"] = "usr_23.txt",
+  ["http://."] = "usr_23.txt",
+  ["http://aspell.net/man-html/Affix-Compression.html"] = "spell.txt",
+  ["http://aspell.net/man-html/Phonetic-Code.html"] = "spell.txt",
+  ["http://canna.sourceforge.jp/"] = "mbyte.txt",
+  ["http://gnuada.sourceforge.net"] = "ft_ada.txt",
+  ["http://lua-users.org/wiki/StringLibraryTutorial"] = "lua.txt",
+  ["http://michael.toren.net/code/"] = "pi_tar.txt",
+  ["http://papp.plan9.de"] = "syntax.txt",
+  ["http://wiki.services.openoffice.org/wiki/Dictionaries"] = "spell.txt",
+  ["http://www.adapower.com"] = "ft_ada.txt",
+  ["http://www.ghostscript.com/"] = "print.txt",
+  ["http://www.jclark.com/"] = "quickfix.txt",
+}
+
 local function tofile(fname, text)
   local f = io.open(fname, 'w')
   if not f then
@@ -278,10 +295,13 @@ local function ignore_invalid(s)
   )
 end
 
-local function ignore_parse_error(s)
+local function ignore_parse_error(s, fname)
-  -- Ignore parse errors for unclosed codespan/optionlink/tag.
+  local helpfile = vim.fs.basename(fname)
-  -- This is common in vimdocs and is treated as plaintext by :help.
+  return (helpfile == 'pi_netrw.txt'
-  return s:find("^[`'|*]")
+    -- Ignore parse errors for unclosed tag.
+    -- This is common in vimdocs and is treated as plaintext by :help.
+    or s:find("^[*]")
+  )
 end
 
 local function has_ancestor(node, ancestor_name)
@@ -323,7 +343,7 @@ local function validate_url(text, fname)
   local ignored = false
   if vim.fs.basename(fname) == 'pi_netrw.txt' then
     ignored = true
-  elseif text:find('http%:') then
+  elseif text:find('http%:') and not exclude_invalid_urls[text] then
     invalid_urls[text] = vim.fs.basename(fname)
   end
   return ignored
@@ -348,7 +368,7 @@ local function visit_validate(root, level, lang_tree, opt, stats)
   end
 
   if node_name == 'ERROR' then
-    if ignore_parse_error(text) then
+    if ignore_parse_error(text, opt.fname) then
       return
     end
     -- Store the raw text to give context to the error report.
@@ -363,7 +383,8 @@ local function visit_validate(root, level, lang_tree, opt, stats)
       end
     end
   elseif node_name == 'url' then
-    validate_url(text, opt.fname)
+    local fixed_url, _ = fix_url(trim(text))
+    validate_url(fixed_url, opt.fname)
   elseif node_name == 'taglink' or node_name == 'optionlink' then
     local _, _, _ = validate_link(root, opt.buf, opt.fname)
   end
@@ -523,7 +544,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     end
     return s
   elseif node_name == 'ERROR' then
-    if ignore_parse_error(trimmed) then
+    if ignore_parse_error(trimmed, opt.fname) then
       return text
     end
 

test/functional/lua/help_spec.lua

@@ -19,10 +19,11 @@ describe(':help docs', function()
     local rv = exec_lua([[return require('scripts.gen_help_html').validate('./build/runtime/doc')]])
     -- Check that we actually found helpfiles.
     ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles)
+    eq({}, rv.invalid_links, 'found invalid :help tag links')
+    eq({}, rv.invalid_urls, 'found invalid URLs in :help docs')
     -- Check that parse errors did not increase wildly.
     -- TODO: Fix all parse errors in :help files.
-    ok(rv.err_count < 100, '<100 parse errors', rv.err_count)
+    ok(rv.err_count < 350, '<350 parse errors', rv.err_count)
-    eq({}, rv.invalid_links, exec_lua([[return 'found invalid :help tag links:\n'..vim.inspect(...)]], rv.invalid_links))
   end)
 
   it('gen_help_html.lua generates HTML', function()
@@ -43,7 +44,7 @@ describe(':help docs', function()
       tmpdir
     )
     eq(4, #rv.helpfiles)
-    ok(rv.err_count == 0, '0 parse errors', rv.err_count)
+    ok(rv.err_count < 25, '<25 parse errors', rv.err_count)
-    eq({}, rv.invalid_links, exec_lua([[return 'found invalid :help tag links:\n'..vim.inspect(...)]], rv.invalid_links))
+    eq({}, rv.invalid_links, 'found invalid :help tag links')
   end)
 end)