doc2tex: generate docs to Latex (#17997)

* `doc2tex`: generate docs to Latex * address some comments
2026-01-17 18:17:11 +00:00 · 2021-05-14 08:30:47 +03:00
parent 3c622d7999
commit 97970d9dcc
12 changed files with 171 additions and 161 deletions
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -444,6 +444,7 @@ proc parseCommand*(command: string): Command =
  of "e": cmdNimscript
  of "doc0": cmdDoc0
  of "doc2", "doc": cmdDoc2
+  of "doc2tex": cmdDoc2tex
  of "rst2html": cmdRst2html
  of "rst2tex": cmdRst2tex
  of "jsondoc0": cmdJsondoc0
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -169,14 +169,17 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
  else:
    result = getOutFile(conf, filename, ext)

-proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil): PDoc =
+proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
+
+proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
+                    outExt: string = HtmlExt, module: PSym = nil): PDoc =
  declareClosures()
  new(result)
  result.module = module
  result.conf = conf
  result.cache = cache
  result.outDir = conf.outDir.string
-  initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
+  initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
                   conf.configVars, filename.string,
                   {roSupportRawDirective, roSupportMarkdown,
                    roPreferMarkdown, roNimFile},
@@ -249,7 +252,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,

 template dispA(conf: ConfigRef; dest: var string, xml, tex: string,
               args: openArray[string]) =
-  if conf.cmd != cmdRst2tex: dest.addf(xml, args)
+  if not conf.isLatexCmd: dest.addf(xml, args)
  else: dest.addf(tex, args)

 proc getVarIdx(varnames: openArray[string], id: string): int =
@@ -551,7 +554,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
        let id = $d.listingCounter
        dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
        var dest2 = ""
-        renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+        renderNimCode(dest2, code, d.target)
        dest.add dest2
        dest.add(d.config.getOrDefault"doc.listing_end" % id)
        return rsRunnable
@@ -821,7 +824,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =

  inc(d.id)
  let
-    plainNameEsc = xmltree.escape(plainName.strip)
+    plainNameEsc = esc(d.target, plainName.strip)
+    uniqueName = if k in routineKinds: plainNameEsc else: name
    cleanPlainSymbol = renderPlainSymbolName(nameNode)
    complexSymbol = complexName(k, n, cleanPlainSymbol)
    plainSymbolEnc = encodeUrl(cleanPlainSymbol)
@@ -835,7 +839,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
  let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)

  d.section[k].add(getConfigVar(d.conf, "doc.item") %
-    ["name", name, "header", result, "desc", comm, "itemID", $d.id,
+    ["name", name, "uniqueName", uniqueName,
+     "header", result, "desc", comm, "itemID", $d.id,
     "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
     "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc,
     "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
@@ -1199,7 +1204,8 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
    var shouldSort = i in routineKinds and groupedToc
    genSection(d, i, shouldSort)
    toc.add(d.toc[i])
-  if toc != "":
+  if toc != "" or d.target == outLatex:
+    # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set
    toc = getConfigVar(d.conf, "doc.toc") % ["content", toc]
  for i in TSymKind: code.add(d.section[i])

@@ -1217,7 +1223,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
        "\\\\\\vspace{0.5em}\\large $1", [d.meta[metaSubtitle]])

  var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
-  let bodyname = if d.hasToc and not d.isPureRst:
+  let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd:
                   groupsection.setLen 0
                   "doc.body_toc_group"
                 elif d.hasToc: "doc.body_toc"
--- a/compiler/docgen2.nim
+++ b/compiler/docgen2.nim
@@ -71,10 +71,15 @@ template myOpenImpl(ext: untyped) {.dirty.} =
 proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
  myOpenImpl(HtmlExt)

+proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+  myOpenImpl(TexExt)
+
 proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
  myOpenImpl(JsonExt)

 const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
+const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
+                                 close = close)
 const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
                                  close = closeJson)

--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -69,12 +69,15 @@ proc commandCheck(graph: ModuleGraph) =
    writeRodFiles(graph)

 when not defined(leanCompiler):
-  proc commandDoc2(graph: ModuleGraph; json: bool) =
+  proc commandDoc2(graph: ModuleGraph; ext: string) =
    handleDocOutputOptions graph.config
    graph.config.setErrorMaxHighMaybe
    semanticPasses(graph)
-    if json: registerPass(graph, docgen2JsonPass)
-    else: registerPass(graph, docgen2Pass)
+    case ext:
+    of TexExt:  registerPass(graph, docgen2TexPass)
+    of JsonExt: registerPass(graph, docgen2JsonPass)
+    of HtmlExt: registerPass(graph, docgen2Pass)
+    else: doAssert false, $ext
    compileProject(graph)
    finishDoc2Pass(graph.config.projectName)

@@ -249,7 +252,8 @@ proc mainCommand*(graph: ModuleGraph) =
      conf.quitOrRaise "compiler wasn't built with documentation generator"
    else:
      wantMainModule(conf)
-      loadConfigs(DocConfig, cache, conf, graph.idgen)
+      let docConf = if conf.cmd == cmdDoc2tex: DocTexConfig else: DocConfig
+      loadConfigs(docConf, cache, conf, graph.idgen)
      defineSymbol(conf.symbols, "nimdoc")
      body

@@ -285,7 +289,7 @@ proc mainCommand*(graph: ModuleGraph) =
        # of labels links in doc comments, e.g. for random.rand:
        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-      commandDoc2(graph, false)
+      commandDoc2(graph, HtmlExt)
      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
        commandBuildIndex(conf, $conf.outDir)
  of cmdRst2html:
@@ -299,7 +303,7 @@ proc mainCommand*(graph: ModuleGraph) =
    else:
      loadConfigs(DocConfig, cache, conf, graph.idgen)
      commandRst2Html(cache, conf)
-  of cmdRst2tex:
+  of cmdRst2tex, cmdDoc2tex:
    for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX,
                 warnLanguageXNotSupported,
                 warnFieldXNotSupported, warnRstStyle]:
@@ -307,10 +311,13 @@ proc mainCommand*(graph: ModuleGraph) =
    when defined(leanCompiler):
      conf.quitOrRaise "compiler wasn't built with documentation generator"
    else:
-      loadConfigs(DocTexConfig, cache, conf, graph.idgen)
-      commandRst2TeX(cache, conf)
+      if conf.cmd == cmdRst2tex:
+        loadConfigs(DocTexConfig, cache, conf, graph.idgen)
+        commandRst2TeX(cache, conf)
+      else:
+        docLikeCmd commandDoc2(graph, TexExt)
  of cmdJsondoc0: docLikeCmd commandJson(cache, conf)
-  of cmdJsondoc: docLikeCmd commandDoc2(graph, true)
+  of cmdJsondoc: docLikeCmd commandDoc2(graph, JsonExt)
  of cmdCtags: docLikeCmd commandTags(cache, conf)
  of cmdBuildindex: docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
  of cmdGendepend: commandGenDepend(graph)
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -144,6 +144,7 @@ type
    cmdNimscript # evaluate nimscript
    cmdDoc0
    cmdDoc2
+    cmdDoc2tex
    cmdRst2html # convert a reStructuredText file to HTML
    cmdRst2tex # convert a reStructuredText file to TeX
    cmdJsondoc0
@@ -160,7 +161,8 @@ type

 const
  cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun}
-  cmdDocLike* = {cmdDoc0, cmdDoc2, cmdJsondoc0, cmdJsondoc, cmdCtags, cmdBuildindex}
+  cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
+                 cmdCtags, cmdBuildindex}

 type
  TStringSeq* = seq[string]
--- a/compiler/renderverbatim.nim
+++ b/compiler/renderverbatim.nim
@@ -1,8 +1,6 @@
 import strutils
-from xmltree import addEscaped

 import ast, options, msgs
-import packages/docutils/highlite

 const isDebug = false
 when isDebug:
@@ -131,22 +129,3 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
      lastNonemptyPos = result.len
  result.setLen lastNonemptyPos

-proc renderNimCode*(result: var string, code: string, isLatex = false) =
-  var toknizr: GeneralTokenizer
-  initGeneralTokenizer(toknizr, code)
-  var buf = ""
-  template append(kind, val) =
-    buf.setLen 0
-    buf.addEscaped(val)
-    let class = tokenClassToStr[kind]
-    if isLatex:
-      result.addf "\\span$1{$2}", [class, buf]
-    else:
-      result.addf  "<span class=\"$1\">$2</span>", [class, buf]
-  while true:
-    getNextToken(toknizr, langNim)
-    case toknizr.kind
-    of gtEof: break  # End Of File (or string)
-    else:
-      # TODO: avoid alloc; maybe toOpenArray
-      append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))
--- a/config/nimdoc.cfg
+++ b/config/nimdoc.cfg
@@ -42,6 +42,7 @@ doc.section.toc2 = """
 # * $itemSymOrID: the symbolic name or the ID if that is not unique.
 # * $itemSymOrIDEnc: quoted version for URLs or attributes.
 # * $name: reduced name of the item.
+# * $uniqueName: name with parameters for routine types or $name for others.
 # * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used).

 doc.item = """
--- a/config/nimdoc.tex.cfg
+++ b/config/nimdoc.tex.cfg
@@ -8,22 +8,24 @@ split.item.toc = "20"
 # after this number of characters

 doc.section = """
-\chapter{$sectionTitle}\label{$sectionID}
-\begin{description}
+\rsthA{$sectionTitle}\label{$sectionID}
 $content
-\end{description}
 """

 doc.section.toc = ""
 # $sectionID $sectionTitleID $sectionTitle $content

 doc.item = """
-\item[\texttt{$header}\label{$itemID}]\mbox{~}\\*
+
+\phantomsection\addcontentsline{toc}{subsection}{$uniqueName}
+
+\begin{rstpre}
+$header
+\end{rstpre}
 $desc
 """

 doc.item.toc = ""
-#  \item $name\ref{$itemID}

 doc.toc = r"\tableofcontents \newpage"

@@ -38,6 +40,10 @@ $moduledesc
 $content
 """

+# $1 - number of listing in document, $2 - language (e.g. langNim), $3 - anchor
+doc.listing_start = "\\begin{rstpre}\n"
+doc.listing_end = "\n\\end{rstpre}\n"
+
 doc.file = """
 % This file was generated by Nim.
 % Generated: $date $time UTC
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -26,7 +26,8 @@
 ## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and
 ##    `nim rst2html`:cmd: for ``*.rst`` files
 ##
-## .. [#latex] command `nim rst2tex`:cmd: for ``*.rst``.
+## .. [#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and
+##    `nim rst2tex`:cmd: for ``*.rst``.
 ##
 ## If you are new to RST please consider reading the following:
 ##
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -179,7 +179,9 @@ proc writeIndexFile*(g: var RstGenerator, outfile: string) =
  ## If the index is empty the file won't be created.
  if g.theIndex.len > 0: writeFile(outfile, g.theIndex)

-proc addXmlChar(dest: var string, c: char) =
+proc addHtmlChar(dest: var string, c: char) =
+  # Escapes HTML characters. Note that single quote ' is not escaped as
+  # &apos; -- unlike XML (for standards pre HTML5 it was even forbidden).
  case c
  of '&': add(dest, "&amp;")
  of '<': add(dest, "&lt;")
@@ -195,26 +197,19 @@ proc addRtfChar(dest: var string, c: char) =
  else: add(dest, c)

 proc addTexChar(dest: var string, c: char) =
+  # Escapes 10 special Latex characters. Note that [, ], and ` are not
+  # considered as such. TODO: neither is @, am I wrong?
  case c
-  of '_': add(dest, "\\_")
-  of '{': add(dest, "\\symbol{123}")
-  of '}': add(dest, "\\symbol{125}")
-  of '[': add(dest, "\\symbol{91}")
-  of ']': add(dest, "\\symbol{93}")
-  of '\\': add(dest, "\\symbol{92}")
-  of '$': add(dest, "\\$")
-  of '&': add(dest, "\\&")
-  of '#': add(dest, "\\#")
-  of '%': add(dest, "\\%")
-  of '~': add(dest, "\\symbol{126}")
-  of '@': add(dest, "\\symbol{64}")
-  of '^': add(dest, "\\symbol{94}")
-  of '`': add(dest, "\\symbol{96}")
+  of '_', '{', '}', '$', '&', '#', '%': add(dest, "\\" & c)
+  # \~ and \^ have a special meaning unless they are followed by {}
+  of '~', '^': add(dest, "\\" & c & "{}")
+  # add {} to avoid gobbling up space by \textbackslash
+  of '\\': add(dest, "\\textbackslash{}")
  else: add(dest, c)

 proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} =
  case target
-  of outHtml:  addXmlChar(dest, c)
+  of outHtml:  addHtmlChar(dest, c)
  of outLatex: addTexChar(dest, c)

 proc addSplitter(target: OutputTarget; dest: var string) {.inline.} =
@@ -987,6 +982,25 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string,
      "</td></tr></tbody></table>" & (
      d.config.getOrDefault"doc.listing_button" % id)

+proc renderCodeLang*(result: var string, lang: SourceLanguage, code: string,
+                     target: OutputTarget) =
+  var g: GeneralTokenizer
+  initGeneralTokenizer(g, code)
+  while true:
+    getNextToken(g, lang)
+    case g.kind
+    of gtEof: break
+    of gtNone, gtWhitespace:
+      add(result, substr(code, g.start, g.length + g.start - 1))
+    else:
+      dispA(target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [
+        esc(target, substr(code, g.start, g.length+g.start-1)),
+        tokenClassToStr[g.kind]])
+  deinitGeneralTokenizer(g)
+
+proc renderNimCode*(result: var string, code: string, target: OutputTarget) =
+  renderCodeLang(result, langNim, code, target)
+
 proc renderCode(d: PDoc, n: PRstNode, result: var string) =
  ## Renders a code (code block or inline code), appending it to `result`.
  ##
@@ -1028,19 +1042,7 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) =
      d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
    for letter in m.text: escChar(d.target, result, letter)
  else:
-    var g: GeneralTokenizer
-    initGeneralTokenizer(g, m.text)
-    while true:
-      getNextToken(g, params.lang)
-      case g.kind
-      of gtEof: break
-      of gtNone, gtWhitespace:
-        add(result, substr(m.text, g.start, g.length + g.start - 1))
-      else:
-        dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [
-          esc(d.target, substr(m.text, g.start, g.length+g.start-1)),
-          tokenClassToStr[g.kind]])
-    deinitGeneralTokenizer(g)
+    renderCodeLang(result, params.lang, m.text, d.target)
  dispA(d.target, result, blockEnd, blockEnd)

 proc renderContainer(d: PDoc, n: PRstNode, result: var string) =
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -208,7 +208,7 @@ constructor.

 this should be shown in utils.html
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="DecNumber">3</span><span class="Operator">*</span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">6</span></pre>ditto
+<pre class="listing"><span class="Identifier">assert</span> <span class="DecNumber">3</span><span class="Operator">*</span><span class="DecNumber">2</span> <span class="Operator">==</span> <span class="DecNumber">6</span></pre>ditto

 </dd>

--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -133,13 +133,13 @@ window.addEventListener('DOMContentLoaded', main);
  <a class="reference reference-toplevel" href="#10" id="60">Consts</a>
  <ul class="simple simple-toc-section">
      <li><a class="reference" href="#C_A"
-    title="C_A = 0x7FF0000000000000&apos;f64">C_A</a></li>
+    title="C_A = 0x7FF0000000000000'f64">C_A</a></li>
  <li><a class="reference" href="#C_B"
-    title="C_B = 0o377&apos;i8">C_B</a></li>
+    title="C_B = 0o377'i8">C_B</a></li>
  <li><a class="reference" href="#C_C"
-    title="C_C = 0o277&apos;i8">C_C</a></li>
+    title="C_C = 0o277'i8">C_C</a></li>
  <li><a class="reference" href="#C_D"
-    title="C_D = 0o177777&apos;i16">C_D</a></li>
+    title="C_D = 0o177777'i16">C_D</a></li>

  </ul>
 </li>
@@ -420,21 +420,21 @@ window.addEventListener('DOMContentLoaded', main);
  
  <p class="module-desc">This is the top level module.
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">testproject</span><span class="Whitespace">
-</span><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">subdir</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">subdir_b</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">utils</span><span class="Whitespace">
-</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="Identifier">bar</span><span class="Punctuation">(</span><span class="DecNumber">3</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="DecNumber">4</span><span class="Punctuation">)</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">7</span><span class="Whitespace">
-</span><span class="Identifier">foo</span><span class="Punctuation">(</span><span class="Identifier">enumValueA</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="Identifier">enumValueB</span><span class="Punctuation">)</span><span class="Whitespace">
-</span><span class="Comment"># bug #11078</span><span class="Whitespace">
-</span><span class="Keyword">for</span><span class="Whitespace"> </span><span class="Identifier">x</span><span class="Whitespace"> </span><span class="Keyword">in</span><span class="Whitespace"> </span><span class="StringLit">&quot;xx&quot;</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">discard</span></pre>top2
+<pre class="listing"><span class="Keyword">import</span> <span class="Identifier">testproject</span>
+<span class="Keyword">import</span> <span class="Identifier">subdir</span> <span class="Operator">/</span> <span class="Identifier">subdir_b</span> <span class="Operator">/</span> <span class="Identifier">utils</span>
+<span class="Identifier">doAssert</span> <span class="Identifier">bar</span><span class="Punctuation">(</span><span class="DecNumber">3</span><span class="Punctuation">,</span> <span class="DecNumber">4</span><span class="Punctuation">)</span> <span class="Operator">==</span> <span class="DecNumber">7</span>
+<span class="Identifier">foo</span><span class="Punctuation">(</span><span class="Identifier">enumValueA</span><span class="Punctuation">,</span> <span class="Identifier">enumValueB</span><span class="Punctuation">)</span>
+<span class="Comment"># bug #11078</span>
+<span class="Keyword">for</span> <span class="Identifier">x</span> <span class="Keyword">in</span> <span class="StringLit">&quot;xx&quot;</span><span class="Punctuation">:</span> <span class="Keyword">discard</span></pre>top2
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">testproject</span><span class="Whitespace">
-</span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;in top2&quot;</span></pre>top2 after
+<pre class="listing"><span class="Keyword">import</span> <span class="Identifier">testproject</span>
+<span class="Keyword">discard</span> <span class="StringLit">&quot;in top2&quot;</span></pre>top2 after
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">testproject</span><span class="Whitespace">
-</span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;in top3&quot;</span></pre>top3 after
+<pre class="listing"><span class="Keyword">import</span> <span class="Identifier">testproject</span>
+<span class="Keyword">discard</span> <span class="StringLit">&quot;in top3&quot;</span></pre>top3 after
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">testproject</span><span class="Whitespace">
-</span><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="DecNumber">3</span><span class="Operator">*</span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">6</span></pre></p>
+<pre class="listing"><span class="Keyword">import</span> <span class="Identifier">testproject</span>
+<span class="Identifier">assert</span> <span class="DecNumber">3</span><span class="Operator">*</span><span class="DecNumber">2</span> <span class="Operator">==</span> <span class="DecNumber">6</span></pre></p>
  <div class="section" id="6">
 <h1><a class="toc-backref" href="#6">Imports</a></h1>
 <dl class="item">
@@ -574,7 +574,7 @@ My someFunc. Stuff in <tt class="docutils literal"><span class="pre"><span class

 came form utils but should be shown where <tt class="docutils literal"><span class="pre"><span class="Identifier">fromUtilsGen</span></span></tt> is called
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">1</span></pre>

 </dd>
 <a id="isValid,T"></a>
@@ -597,7 +597,7 @@ cz1

 cz2
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;in cz2&quot;</span></pre>
+<pre class="listing"><span class="Keyword">discard</span> <span class="StringLit">&quot;in cz2&quot;</span></pre>

 </dd>
 <a id="z3"></a>
@@ -648,7 +648,7 @@ cz8


 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">+</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">2</span></pre>
+<pre class="listing"><span class="Identifier">doAssert</span> <span class="DecNumber">1</span> <span class="Operator">+</span> <span class="DecNumber">1</span> <span class="Operator">==</span> <span class="DecNumber">2</span></pre>

 </dd>
 <a id="z10"></a>
@@ -657,7 +657,7 @@ cz8


 <p><strong class="examples_text">Example: cmd: -d:foobar</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>cz10
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">1</span></pre>cz10

 </dd>
 <a id="z11"></a>
@@ -666,7 +666,7 @@ cz8


 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">1</span></pre>

 </dd>
 <a id="z12"></a>
@@ -675,7 +675,7 @@ cz8


 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">1</span></pre>

 </dd>
 <a id="z13"></a>
@@ -700,7 +700,7 @@ cz13

 cz17 rest
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>rest
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">1</span></pre>rest

 </dd>
 <a id="p1"></a>
@@ -709,21 +709,21 @@ cz17 rest

 cp1
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Comment"># regular comments work here</span></pre>c4
+<pre class="listing"><span class="Identifier">doAssert</span> <span class="DecNumber">1</span> <span class="Operator">==</span> <span class="DecNumber">1</span> <span class="Comment"># regular comments work here</span></pre>c4
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Comment"># c5 regular comments before 1st token work</span><span class="Whitespace">
-</span><span class="Comment"># regular comment</span><span class="Whitespace">
-</span><span class="LongComment">#[
+<pre class="listing"><span class="Comment"># c5 regular comments before 1st token work</span>
+<span class="Comment"># regular comment</span>
+<span class="LongComment">#[
 nested regular comment
-]#</span><span class="Whitespace">
-</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Comment"># c8</span><span class="Whitespace">
-</span><span class="Comment">## this is a non-nested doc comment</span><span class="Whitespace">
+]#</span>
+<span class="Identifier">doAssert</span> <span class="DecNumber">2</span> <span class="Operator">==</span> <span class="DecNumber">2</span> <span class="Comment"># c8</span>
+<span class="Comment">## this is a non-nested doc comment</span>

-</span><span class="LongComment">##[
+<span class="LongComment">##[
 this is a nested doc comment
-]##</span><span class="Whitespace">
-</span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;c9&quot;</span><span class="Whitespace">
-</span><span class="Comment"># also work after</span></pre>
+]##</span>
+<span class="Keyword">discard</span> <span class="StringLit">&quot;c9&quot;</span>
+<span class="Comment"># also work after</span></pre>

 </dd>
 <a id="addfBug14485"></a>
@@ -732,9 +732,9 @@ this is a nested doc comment

 Some proc
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo() = &quot;</span><span class="Whitespace"> </span><span class="Operator">&amp;</span><span class="Whitespace"> </span><span class="Operator">$</span><span class="Punctuation">[</span><span class="DecNumber">1</span><span class="Punctuation">]</span><span class="Whitespace">
-</span><span class="LongComment">#[
-0: let&apos;s also add some broken html to make sure this won&apos;t break in future
+<pre class="listing"><span class="Keyword">discard</span> <span class="StringLit">&quot;foo() = &quot;</span> <span class="Operator">&amp;</span> <span class="Operator">$</span><span class="Punctuation">[</span><span class="DecNumber">1</span><span class="Punctuation">]</span>
+<span class="LongComment">#[
+0: let's also add some broken html to make sure this won't break in future
 1: &lt;/span&gt;
 2: &lt;/span&gt;
 3: &lt;/span
@@ -782,7 +782,7 @@ the c printf. etc.
 </ul>
 <pre class="listing"><span class="Identifier">low2</span><span class="Punctuation">(</span><span class="DecNumber">2</span><span class="Punctuation">)</span> <span class="Comment"># =&gt; -9223372036854775808</span></pre>
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;in low2&quot;</span></pre>
+<pre class="listing"><span class="Keyword">discard</span> <span class="StringLit">&quot;in low2&quot;</span></pre>

 </dd>
 <a id="tripleStrLitTest"></a>
@@ -791,41 +791,41 @@ the c printf. etc.


 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Comment">## mullitline string litterals are tricky as their indentation can span</span><span class="Whitespace">
-</span><span class="Comment">## below that of the runnableExamples</span><span class="Whitespace">
-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s1a</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;
+<pre class="listing"><span class="Comment">## mullitline string litterals are tricky as their indentation can span</span>
+<span class="Comment">## below that of the runnableExamples</span>
+<span class="Keyword">let</span> <span class="Identifier">s1a</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot;
 should appear at indent 0
  at indent 2
 at indent 0
-&quot;&quot;&quot;</span><span class="Whitespace">
-</span><span class="Comment"># make sure this works too</span><span class="Whitespace">
-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s1b</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;start at same line
+&quot;&quot;&quot;</span>
+<span class="Comment"># make sure this works too</span>
+<span class="Keyword">let</span> <span class="Identifier">s1b</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot;start at same line
  at indent 2
 at indent 0
-&quot;&quot;&quot;</span><span class="Whitespace"> </span><span class="Comment"># comment after</span><span class="Whitespace">
-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s2</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;sandwich &quot;&quot;&quot;</span><span class="Whitespace">
-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s3</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;</span><span class="Whitespace">
-</span><span class="Keyword">when</span><span class="Whitespace"> </span><span class="Identifier">false</span><span class="Punctuation">:</span><span class="Whitespace">
-  </span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s5</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;
-        in s5 &quot;&quot;&quot;</span><span class="Whitespace">
+&quot;&quot;&quot;</span> <span class="Comment"># comment after</span>
+<span class="Keyword">let</span> <span class="Identifier">s2</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot;sandwich &quot;&quot;&quot;</span>
+<span class="Keyword">let</span> <span class="Identifier">s3</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;</span>
+<span class="Keyword">when</span> <span class="Identifier">false</span><span class="Punctuation">:</span>
+  <span class="Keyword">let</span> <span class="Identifier">s5</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot;
+        in s5 &quot;&quot;&quot;</span>

-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s3b</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="Punctuation">[</span><span class="LongStringLit">&quot;&quot;&quot;
+<span class="Keyword">let</span> <span class="Identifier">s3b</span> <span class="Operator">=</span> <span class="Punctuation">[</span><span class="LongStringLit">&quot;&quot;&quot;
 %!? #[...] # inside a multiline ...
-&quot;&quot;&quot;</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo&quot;</span><span class="Punctuation">]</span><span class="Whitespace">
+&quot;&quot;&quot;</span><span class="Punctuation">,</span> <span class="StringLit">&quot;foo&quot;</span><span class="Punctuation">]</span>

-</span><span class="Comment">## make sure handles trailing spaces</span><span class="Whitespace">
-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s4</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot; 
-&quot;&quot;&quot;</span><span class="Whitespace">
+<span class="Comment">## make sure handles trailing spaces</span>
+<span class="Keyword">let</span> <span class="Identifier">s4</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot; 
+&quot;&quot;&quot;</span>

-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s5</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot; x
-&quot;&quot;&quot;</span><span class="Whitespace">
-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s6</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot; &quot;&quot;
-&quot;&quot;&quot;</span><span class="Whitespace">
-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s7</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;</span><span class="Whitespace">
-</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s8</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="Punctuation">[</span><span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;
-  &quot;&quot;&quot;</span><span class="Whitespace"> </span><span class="Punctuation">]</span><span class="Whitespace">
-</span><span class="Keyword">discard</span><span class="Whitespace">
-</span><span class="Comment"># should be in</span></pre>
+<span class="Keyword">let</span> <span class="Identifier">s5</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot; x
+&quot;&quot;&quot;</span>
+<span class="Keyword">let</span> <span class="Identifier">s6</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot; &quot;&quot;
+&quot;&quot;&quot;</span>
+<span class="Keyword">let</span> <span class="Identifier">s7</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;</span>
+<span class="Keyword">let</span> <span class="Identifier">s8</span> <span class="Operator">=</span> <span class="Punctuation">[</span><span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;</span><span class="Punctuation">,</span> <span class="LongStringLit">&quot;&quot;&quot;
+  &quot;&quot;&quot;</span> <span class="Punctuation">]</span>
+<span class="Keyword">discard</span>
+<span class="Comment"># should be in</span></pre>

 </dd>
 <a id="asyncFun1"></a>
@@ -896,9 +896,9 @@ foo3


 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Comment"># ok1</span><span class="Whitespace">
-</span><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace">
-</span><span class="Comment"># ok2</span></pre>
+<pre class="listing"><span class="Comment"># ok1</span>
+<span class="Identifier">assert</span> <span class="DecNumber">1</span> <span class="Operator">==</span> <span class="DecNumber">1</span>
+<span class="Comment"># ok2</span></pre>

 </dd>
 <a id="iter1.i,int"></a>
@@ -914,7 +914,7 @@ foo1

 foo2
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="Comment"># bar</span></pre>
+<pre class="listing"><span class="Keyword">discard</span> <span class="Comment"># bar</span></pre>

 </dd>

@@ -935,9 +935,9 @@ foo2


 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>cz16 after
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">1</span></pre>cz16 after
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">+</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>
+<pre class="listing"><span class="Identifier">doAssert</span> <span class="DecNumber">2</span> <span class="Operator">==</span> <span class="DecNumber">1</span> <span class="Operator">+</span> <span class="DecNumber">1</span></pre>

 </dd>
 <a id="z18.m"></a>
@@ -979,20 +979,20 @@ This does nothing


 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">std</span><span class="Operator">/</span><span class="Identifier">strutils</span><span class="Whitespace">
-</span><span class="Comment">## issue #8871 preserve formatting</span><span class="Whitespace">
-</span><span class="Comment">## line doc comment</span><span class="Whitespace">
-</span><span class="Comment"># bar</span><span class="Whitespace">
-</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace">
-</span><span class="LongComment">##[
+<pre class="listing"><span class="Keyword">import</span> <span class="Identifier">std</span><span class="Operator">/</span><span class="Identifier">strutils</span>
+<span class="Comment">## issue #8871 preserve formatting</span>
+<span class="Comment">## line doc comment</span>
+<span class="Comment"># bar</span>
+<span class="Identifier">doAssert</span> <span class="StringLit">&quot;'foo&quot;</span> <span class="Operator">==</span> <span class="StringLit">&quot;'foo&quot;</span>
+<span class="LongComment">##[
 foo
 bar
-]##</span><span class="Whitespace">
+]##</span>

-</span><span class="Identifier">doAssert</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">not</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo&quot;</span><span class="Operator">.</span><span class="Identifier">startsWith</span><span class="Whitespace"> </span><span class="StringLit">&quot;ba&quot;</span><span class="Whitespace">
-</span><span class="Keyword">block</span><span class="Punctuation">:</span><span class="Whitespace">
-  </span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="HexNumber">0xff</span><span class="Whitespace"> </span><span class="Comment"># elu par cette crapule</span><span class="Whitespace">
-</span><span class="Comment"># should be in</span></pre>should be still in
+<span class="Identifier">doAssert</span><span class="Punctuation">:</span> <span class="Keyword">not</span> <span class="StringLit">&quot;foo&quot;</span><span class="Operator">.</span><span class="Identifier">startsWith</span> <span class="StringLit">&quot;ba&quot;</span>
+<span class="Keyword">block</span><span class="Punctuation">:</span>
+  <span class="Keyword">discard</span> <span class="HexNumber">0xff</span> <span class="Comment"># elu par cette crapule</span>
+<span class="Comment"># should be in</span></pre>should be still in

 </dd>
 <a id="z14.t"></a>
@@ -1012,13 +1012,13 @@ cz15
 <p><strong class="examples_text">Example:</strong></p>
 <pre class="listing"><span class="Keyword">discard</span></pre>
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">3</span></pre>
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">3</span></pre>
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">4</span></pre>ok5 ok5b
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">4</span></pre>ok5 ok5b
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="Identifier">true</span></pre>
+<pre class="listing"><span class="Identifier">assert</span> <span class="Identifier">true</span></pre>
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>in or out?
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">1</span></pre>in or out?

 </dd>
 <a id="testNimDocTrailingExample.t"></a>
@@ -1027,7 +1027,7 @@ cz15


 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">2</span></pre>
+<pre class="listing"><span class="Keyword">discard</span> <span class="DecNumber">2</span></pre>

 </dd>