docgen: move to shared RST state (fix #16990) (#18256)

* docgen: move to shared RST state (fix #16990)

* Update lib/packages/docutils/rst.nim

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

* Update lib/packages/docutils/rst.nim

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

* Update lib/packages/docutils/rst.nim

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* rename `cmdDoc2` to `cmdDoc`

* fix (P)RstSharedState convention

* new style of initialization

* misc suggestions

* 1 more rename

* fix a regression

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>
Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

compiler/commands.nim

@@ -443,7 +443,7 @@ proc parseCommand*(command: string): Command =
   of "check": cmdCheck
   of "e": cmdNimscript
   of "doc0": cmdDoc0
-  of "doc2", "doc": cmdDoc2
+  of "doc2", "doc": cmdDoc
   of "doc2tex": cmdDoc2tex
   of "rst2html": cmdRst2html
   of "rst2tex": cmdRst2tex

compiler/docgen.nim

@@ -28,6 +28,24 @@ const
   DocColOffset = "## ".len  # assuming that a space was added after ##
 
 type
+  ItemFragment = object  ## A fragment from each item will be eventually
+                         ## constructed by converting `rst` fields to strings.
+    case isRst: bool
+    of true:
+      rst: PRstNode
+    of false:            ## contains ready markup e.g. from runnableExamples
+      str: string
+  ItemPre = seq[ItemFragment]  ## A pre-processed item.
+  Item = object        ## Any item in documentation, e.g. symbol
+                       ## entry. Configuration variable ``doc.item``
+                       ## is used for its HTML rendering.
+    descRst: ItemPre     ## Description of the item (may contain
+                         ## runnableExamples).
+    substitutions: seq[string]    ## Variable names in `doc.item`...
+  ModSection = object  ## Section like Procs, Types, etc.
+    secItems: seq[Item]  ## Pre-processed items.
+    finalMarkup: string  ## The items, after RST pass 2 and rendering.
+  ModSections = array[TSymKind, ModSection]
   TSections = array[TSymKind, string]
   ExampleGroup = ref object
     ## a group of runnableExamples with same rdoccmd
@@ -35,17 +53,25 @@ type
     docCmd: string ## from user config, e.g. --doccmd:-d:foo
     code: string ## contains imports; each import contains `body`
     index: int ## group index
+  JsonItem = object  # pre-processed item: `rst` should be finalized
+    json: JsonNode
+    rst: PRstNode
+    rstField: string
   TDocumentor = object of rstgen.RstGenerator
-    modDesc: string       # module description
+    modDescPre: ItemPre   # module description, not finalized
+    modDescFinal: string  # module description, after RST pass 2 and rendering
     module: PSym
     modDeprecationMsg: string
-    toc, toc2, section: TSections
+    section: ModSections     # entries of ``.nim`` file (for `proc`s, etc)
+    toc, toc2: TSections     # toc2 - grouped TOC
     tocTable: array[TSymKind, Table[string, string]]
     indexValFilename: string
     analytics: string  # Google Analytics javascript, "" if doesn't exist
     seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML.
-    jArray: JsonNode
+    jEntriesPre: seq[JsonItem] # pre-processed RST + JSON content
+    jEntriesFinal: JsonNode    # final JSON after RST pass 2 and rendering
     types: TStrTable
+    sharedState: PRstSharedState
     isPureRst: bool
     conf*: ConfigRef
     cache*: IdentCache
@@ -58,6 +84,9 @@ type
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
+proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst)
+proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str)
+
 proc prettyString(a: object): string =
   # xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906
   for k, v in fieldPairs(a):
@@ -151,12 +180,12 @@ template declareClosures =
       if not fileExists(result): result = ""
 
 proc parseRst(text, filename: string,
-              line, column: int, hasToc: var bool,
+              line, column: int,
               rstOptions: RstParseOptions;
-              conf: ConfigRef): PRstNode =
+              conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
   declareClosures()
-  result = rstParse(text, filename, line, column, hasToc, rstOptions,
-                    docgenFindFile, compilerMsgHandler)
+  result = rstParsePass1(text, filename, line, column, rstOptions,
+                         sharedState)
 
 proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
                  ext: string, guessTarget: bool): AbsoluteFile =
@@ -179,10 +208,13 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.conf = conf
   result.cache = cache
   result.outDir = conf.outDir.string
+  const options = {roSupportRawDirective, roSupportMarkdown,
+                   roPreferMarkdown, roNimFile}
+  result.sharedState = newRstSharedState(
+      options, filename.string,
+      docgenFindFile, compilerMsgHandler)
   initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
-                   conf.configVars, filename.string,
-                   {roSupportRawDirective, roSupportMarkdown,
-                    roPreferMarkdown, roNimFile},
+                   conf.configVars, filename.string, options,
                    docgenFindFile, compilerMsgHandler)
 
   if conf.configVars.hasKey("doc.googleAnalytics"):
@@ -203,7 +235,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
 
   result.seenSymbols = newStringTable(modeCaseInsensitive)
   result.id = 100
-  result.jArray = newJArray()
+  result.jEntriesFinal = newJArray()
   initStrTable result.types
   result.onTestSnippet =
     proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
@@ -261,37 +293,31 @@ proc getVarIdx(varnames: openArray[string], id: string): int =
       return i
   result = -1
 
-proc genComment(d: PDoc, n: PNode): string =
-  result = ""
+proc genComment(d: PDoc, n: PNode): PRstNode =
   if n.comment.len > 0:
-    let comment = n.comment
-    when false:
-      # RFC: to preseve newlines in comments, this would work:
-      comment = comment.replace("\n", "\n\n")
-    renderRstToOut(d[],
-                   parseRst(comment, toFullPath(d.conf, n.info),
-                            toLinenumber(n.info),
-                            toColumn(n.info) + DocColOffset,
-                            (var dummy: bool; dummy), d.options, d.conf),
-                   result)
+    result = parseRst(n.comment, toFullPath(d.conf, n.info),
+                      toLinenumber(n.info),
+                      toColumn(n.info) + DocColOffset,
+                      d.options, d.conf,
+                      d.sharedState)
 
-proc genRecCommentAux(d: PDoc, n: PNode): string =
-  if n == nil: return ""
+proc genRecCommentAux(d: PDoc, n: PNode): PRstNode =
+  if n == nil: return nil
   result = genComment(d, n)
-  if result == "":
+  if result == nil:
     if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef,
                   nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}:
       # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}:
       for i in 0..<n.len:
         result = genRecCommentAux(d, n[i])
-        if result != "": return
+        if result != nil: return
   else:
     n.comment = ""
 
-proc genRecComment(d: PDoc, n: PNode): string =
-  if n == nil: return ""
+proc genRecComment(d: PDoc, n: PNode): PRstNode =
+  if n == nil: return nil
   result = genComment(d, n)
-  if result == "":
+  if result == nil:
     if n.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
                   nkMacroDef, nkTemplateDef, nkConverterDef}:
       result = genRecCommentAux(d, n[bodyPos])
@@ -519,7 +545,7 @@ type RunnableState = enum
   rsRunnable
   rsDone
 
-proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
+proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var ItemPre,
                                 state: RunnableState, topLevel: bool):
                                RunnableState =
   ##[
@@ -550,8 +576,10 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
         let (rdoccmd, code) = prepareExample(d, n, topLevel)
         var msg = "Example:"
         if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
-        dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
+        var s: string
+        dispA(d.conf, s, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
             "\n\n\\textbf{$1}\n", [msg])
+        dest.add s
         inc d.listingCounter
         let id = $d.listingCounter
         dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
@@ -597,7 +625,7 @@ proc getRoutineBody(n: PNode): PNode =
     doAssert result.len == 2
     result = result[1]
 
-proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var string) =
+proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var ItemPre) =
   var n = n
   var state = rsStart
   template fn(n2, topLevel) =
@@ -706,7 +734,7 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string =
   ## types will be added with a preceding dash. Return types won't be added.
   ##
   ## If you modify the output of this proc, please update the anchor generation
-  ## section of ``doc/docgen.txt``.
+  ## section of ``doc/docgen.rst``.
   result = baseName
   case k
   of skProc, skFunc: discard
@@ -804,7 +832,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   var result = ""
   var literal, plainName = ""
   var kind = tkEof
-  var comm = ""
+  var comm: ItemPre
   if n.kind in routineDefs:
     getAllRunnableExamples(d, n, comm)
   else:
@@ -840,9 +868,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
 
   let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
 
-  d.section[k].add(getConfigVar(d.conf, "doc.item") %
-    ["name", name, "uniqueName", uniqueName,
-     "header", result, "desc", comm, "itemID", $d.id,
+  d.section[k].secItems.add Item(descRst: comm, substitutions: @[
+     "name", name, "uniqueName", uniqueName,
+     "header", result, "itemID", $d.id,
      "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
      "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc,
      "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
@@ -886,25 +914,26 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   if k == skType and nameNode.kind == nkSym:
     d.types.strTableAdd nameNode.sym
 
-proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode =
+proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem =
   if not isVisible(d, nameNode): return
   var
     name = getName(d, nameNode)
     comm = genRecComment(d, n)
     r: TSrcGen
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
-  result = %{ "name": %name, "type": %($k), "line": %n.info.line.int,
-                 "col": %n.info.col}
-  if comm.len > 0:
-    result["description"] = %comm
+  result.json = %{ "name": %name, "type": %($k), "line": %n.info.line.int,
+                   "col": %n.info.col}
+  if comm != nil:
+    result.rst = comm
+    result.rstField = "description"
   if r.buf.len > 0:
-    result["code"] = %r.buf
+    result.json["code"] = %r.buf
   if k in routineKinds:
-    result["signature"] = newJObject()
+    result.json["signature"] = newJObject()
     if n[paramsPos][0].kind != nkEmpty:
-      result["signature"]["return"] = %($n[paramsPos][0])
+      result.json["signature"]["return"] = %($n[paramsPos][0])
     if n[paramsPos].len > 1:
-      result["signature"]["arguments"] = newJArray()
+      result.json["signature"]["arguments"] = newJArray()
     for paramIdx in 1 ..< n[paramsPos].len:
       for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2:
         let
@@ -912,22 +941,22 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode =
           paramType = $n[paramsPos][paramIdx][^2]
         if n[paramsPos][paramIdx][^1].kind != nkEmpty:
           let paramDefault = $n[paramsPos][paramIdx][^1]
-          result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault}
+          result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault}
         else:
-          result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType}
+          result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType}
     if n[pragmasPos].kind != nkEmpty:
-      result["signature"]["pragmas"] = newJArray()
+      result.json["signature"]["pragmas"] = newJArray()
       for pragma in n[pragmasPos]:
-        result["signature"]["pragmas"].add %($pragma)
+        result.json["signature"]["pragmas"].add %($pragma)
     if n[genericParamsPos].kind != nkEmpty:
-      result["signature"]["genericParams"] = newJArray()
+      result.json["signature"]["genericParams"] = newJArray()
       for genericParam in n[genericParamsPos]:
         var param = %{"name": %($genericParam)}
         if genericParam.sym.typ.sons.len > 0:
           param["types"] = newJArray()
         for kind in genericParam.sym.typ.sons:
           param["types"].add %($kind)
-        result["signature"]["genericParams"].add param
+        result.json["signature"]["genericParams"].add param
 
 proc checkForFalse(n: PNode): bool =
   result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
@@ -946,8 +975,8 @@ proc traceDeps(d: PDoc, it: PNode) =
       traceDeps(d, a)
   elif it.kind == nkSym and belongsToPackage(d.conf, it.sym):
     let external = externalDep(d, it.sym)
-    if d.section[k] != "": d.section[k].add(", ")
-    dispA(d.conf, d.section[k],
+    if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
+    dispA(d.conf, d.section[k].finalMarkup,
           "<a class=\"reference external\" href=\"$2\">$1</a>",
           "$1", [esc(d.target, external.prettyLink),
                  changeFileExt(external, "html")])
@@ -956,8 +985,8 @@ proc exportSym(d: PDoc; s: PSym) =
   const k = exportSection
   if s.kind == skModule and belongsToPackage(d.conf, s):
     let external = externalDep(d, s)
-    if d.section[k] != "": d.section[k].add(", ")
-    dispA(d.conf, d.section[k],
+    if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
+    dispA(d.conf, d.section[k].finalMarkup,
           "<a class=\"reference external\" href=\"$2\">$1</a>",
           "$1", [esc(d.target, external.prettyLink),
                  changeFileExt(external, "html")])
@@ -968,9 +997,9 @@ proc exportSym(d: PDoc; s: PSym) =
         complexSymbol = complexName(s.kind, s.ast, s.name.s)
         symbolOrId = d.newUniquePlainSymbol(complexSymbol)
         external = externalDep(d, module)
-      if d.section[k] != "": d.section[k].add(", ")
+      if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
       # XXX proper anchor generation here
-      dispA(d.conf, d.section[k],
+      dispA(d.conf, d.section[k].finalMarkup,
             "<a href=\"$2#$3\"><span class=\"Identifier\">$1</span></a>",
             "$1", [esc(d.target, s.name.s),
                    changeFileExt(external, "html"),
@@ -1034,13 +1063,16 @@ proc documentRaises*(cache: IdentCache; n: PNode) =
     if p5 != nil: n[pragmasPos].add p5
 
 proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
+  ## Goes through nim nodes recursively and collects doc comments.
+  ## Main function for `doc`:option: command,
+  ## which is implemented in ``docgen2.nim``.
   template genItemAux(skind) =
     genItem(d, n, n[namePos], skind, docFlags)
   case n.kind
   of nkPragma:
     let pragmaNode = findPragma(n, wDeprecated)
     d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode))
-  of nkCommentStmt: d.modDesc.add(genComment(d, n))
+  of nkCommentStmt: d.modDescPre.add(genComment(d, n))
   of nkProcDef, nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
     genItemAux(skProc)
@@ -1079,21 +1111,63 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
   of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept"
   of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0])
   of nkCallKinds:
-    var comm = ""
+    var comm: ItemPre
     getAllRunnableExamples(d, n, comm)
-    if comm != "": d.modDesc.add(comm)
+    if comm.len != 0: d.modDescPre.add(comm)
   else: discard
 
-proc add(d: PDoc; j: JsonNode) =
-  if j != nil: d.jArray.add j
+proc finishGenerateDoc*(d: var PDoc) =
+  ## Perform 2nd RST pass for resolution of links/footnotes/headings...
+  # Main title/subtitle are allowed only in the first RST fragment of document
+  var firstRst = PRstNode(nil)
+  for fragment in d.modDescPre:
+    if fragment.isRst:
+      firstRst = fragment.rst
+      break
+  preparePass2(d.sharedState, firstRst)
+
+  # Finalize fragments of ``.nim`` or ``.rst`` file
+  proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) =
+    for f in fragments:
+      case f.isRst:
+      of true:
+        var resolved = resolveSubs(d.sharedState, f.rst)
+        renderRstToOut(d[], resolved, result)
+      of false: result &= f.str
+  for k in TSymKind:
+    for item in d.section[k].secItems:
+      var itemDesc: string
+      renderItemPre(d, item.descRst, itemDesc)
+      d.section[k].finalMarkup.add(
+        getConfigVar(d.conf, "doc.item") % (
+            item.substitutions & @["desc", itemDesc]))
+      itemDesc = ""
+    d.section[k].secItems.setLen 0
+  renderItemPre(d, d.modDescPre, d.modDescFinal)
+  d.modDescPre.setLen 0
+  d.hasToc = d.hasToc or d.sharedState.hasToc
+
+  # Finalize fragments of ``.json`` file
+  for i, entry in d.jEntriesPre:
+    if entry.rst != nil:
+      let resolved = resolveSubs(d.sharedState, entry.rst)
+      var str: string
+      renderRstToOut(d[], resolved, str)
+      entry.json[entry.rstField] = %str
+      d.jEntriesFinal.add entry.json
+      d.jEntriesPre[i].rst = nil
+
+proc add(d: PDoc; j: JsonItem) =
+  if j.json != nil or j.rst != nil: d.jEntriesPre.add j
 
 proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
   case n.kind
   of nkCommentStmt:
     if includeComments:
-      d.add %*{"comment": genComment(d, n)}
+      d.add JsonItem(rst: genComment(d, n), rstField: "comment",
+                     json: %Table[string, string]())
     else:
-      d.modDesc.add(genComment(d, n))
+      d.modDescPre.add(genComment(d, n))
   of nkProcDef, nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
     d.add genJsonItem(d, n, n[namePos], skProc)
@@ -1173,11 +1247,11 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) =
     "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs",
     "Methods", "Iterators", "Converters", "Macros", "Templates", "Exports"
   ]
-  if d.section[kind] == "": return
+  if d.section[kind].finalMarkup == "": return
   var title = sectionNames[kind]
-  d.section[kind] = getConfigVar(d.conf, "doc.section") % [
+  d.section[kind].finalMarkup = getConfigVar(d.conf, "doc.section") % [
       "sectionid", $ord(kind), "sectionTitle", title,
-      "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind]]
+      "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind].finalMarkup]
 
   var tocSource = d.toc
   if groupedToc:
@@ -1209,7 +1283,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
   if toc != "" or d.target == outLatex:
     # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set
     toc = getConfigVar(d.conf, "doc.toc") % ["content", toc]
-  for i in TSymKind: code.add(d.section[i])
+  for i in TSymKind: code.add(d.section[i].finalMarkup)
 
   # Extract the title. Non API modules generate an entry in the index table.
   if d.meta[metaTitle].len != 0:
@@ -1234,7 +1308,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
   let seeSrc = genSeeSrc(d, d.filename, 1)
   content = getConfigVar(d.conf, bodyname) % [
       "title", title, "subtitle", subtitle,
-      "tableofcontents", toc, "moduledesc", d.modDesc, "date", getDateStr(),
+      "tableofcontents", toc, "moduledesc", d.modDescFinal, "date", getDateStr(),
       "time", getClockStr(), "content", code,
       "deprecationMsg", d.modDeprecationMsg,
       "theindexhref", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
@@ -1248,7 +1322,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
         "dochackjs", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
                              docHackJsFname.RelativeFile),
         "title", title, "subtitle", subtitle, "tableofcontents", toc,
-        "moduledesc", d.modDesc, "date", getDateStr(), "time", getClockStr(),
+        "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(),
         "content", content, "author", d.meta[metaAuthor],
         "version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics,
         "deprecationMsg", d.modDeprecationMsg]
@@ -1297,12 +1371,12 @@ proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) =
 proc writeOutputJson*(d: PDoc, useWarning = false) =
   runAllExamples(d)
   var modDesc: string
-  for desc in d.modDesc:
+  for desc in d.modDescFinal:
     modDesc &= desc
   let content = %*{"orig": d.filename,
     "nimble": getPackageName(d.conf, d.filename),
     "moduleDescription": modDesc,
-    "entries": d.jArray}
+    "entries": d.jEntriesFinal}
   if optStdout in d.conf.globalOptions:
     write(stdout, $content)
   else:
@@ -1324,12 +1398,14 @@ proc handleDocOutputOptions*(conf: ConfigRef) =
     conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
 
 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
+  ## implementation of deprecated ``doc0`` command (without semantic checking)
   handleDocOutputOptions conf
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
   var d = newDocumentor(conf.projectFull, cache, conf)
   d.hasToc = true
   generateDoc(d, ast, ast)
+  finishGenerateDoc(d)
   writeOutput(d)
   generateIndex(d)
 
@@ -1339,14 +1415,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef;
   var d = newDocumentor(filen, cache, conf, outExt)
 
   d.isPureRst = true
-  var rst = parseRst(readFile(filen.string), filen.string,
+  let rst = parseRst(readFile(filen.string), filen.string,
                      line=LineRstInit, column=ColRstInit,
-                     d.hasToc,
-                     {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown},
-                     conf)
-  var modDesc = newStringOfCap(30_000)
-  renderRstToOut(d[], rst, modDesc)
-  d.modDesc = modDesc
+                     {roSupportRawDirective, roSupportMarkdown,
+                      roPreferMarkdown}, conf,
+                     d.sharedState)
+  d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
+  finishGenerateDoc(d)
   writeOutput(d)
   generateIndex(d)
 
@@ -1357,6 +1432,7 @@ proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) =
   commandRstAux(cache, conf, conf.projectFull, TexExt)
 
 proc commandJson*(cache: IdentCache, conf: ConfigRef) =
+  ## implementation of a deprecated jsondoc0 command
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
   var d = newDocumentor(conf.projectFull, cache, conf)
@@ -1366,7 +1442,8 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) =
                warnUser, "the ':test:' attribute is not supported by this backend")
   d.hasToc = true
   generateJson(d, ast)
-  let json = d.jArray
+  finishGenerateDoc(d)
+  let json = d.jEntriesFinal
   let content = pretty(json)
 
   if optStdout in d.conf.globalOptions:

compiler/docgen2.nim

@@ -31,6 +31,7 @@ template closeImpl(body: untyped) {.dirty.} =
   let useWarning = sfMainModule notin g.module.flags
   let groupedToc = true
   if shouldProcess(g):
+    finishGenerateDoc(g.doc)
     body
     try:
       generateIndex(g.doc)

compiler/main.nim

@@ -282,7 +282,7 @@ proc mainCommand*(graph: ModuleGraph) =
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
   of cmdDoc0: docLikeCmd commandDoc(cache, conf)
-  of cmdDoc2:
+  of cmdDoc:
     docLikeCmd():
       conf.setNoteDefaults(warnLockLevel, false) # issue #13218
       conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218

compiler/options.nim

@@ -143,8 +143,8 @@ type
     cmdIdeTools # ide tools (e.g. nimsuggest)
     cmdNimscript # evaluate nimscript
     cmdDoc0
-    cmdDoc2
-    cmdDoc2tex
+    cmdDoc      # convert .nim doc comments to HTML
+    cmdDoc2tex  # convert .nim doc comments to LaTeX
     cmdRst2html # convert a reStructuredText file to HTML
     cmdRst2tex # convert a reStructuredText file to TeX
     cmdJsondoc0
@@ -161,7 +161,7 @@ type
 
 const
   cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun}
-  cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
+  cmdDocLike* = {cmdDoc0, cmdDoc, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
                  cmdCtags, cmdBuildindex}
 
 type

lib/packages/docutils/rst.nim

@@ -441,7 +441,7 @@ proc rawGetTok(L: var Lexer, tok: var Token) =
       inc L.col
   tok.col = max(tok.col - L.baseIndent, 0)
 
-proc getTokens(buffer: string, tokens: var TokenSeq): int =
+proc getTokens(buffer: string, tokens: var TokenSeq) =
   var L: Lexer
   var length = tokens.len
   L.buf = cstring(buffer)
@@ -489,7 +489,7 @@ type
     autoSymIdx: int     # order of occurence: fnAutoSymbol
     label: string       # valid for fnAutoNumberLabel
 
-  SharedState = object
+  RstSharedState = object
     options: RstParseOptions    # parsing options
     hLevels: LevelMap           # hierarchy of heading styles
     hTitleCnt: int              # =0 if no title, =1 if only main title,
@@ -498,8 +498,8 @@ type
     currRole: string            # current interpreted text role
     currRoleKind: RstNodeKind   # ... and its node kind
     subs: seq[Substitution]     # substitutions
-    refs: seq[Substitution]     # references
-    anchors: seq[AnchorSubst]   # internal target substitutions
+    refs*: seq[Substitution]    # references
+    anchors*: seq[AnchorSubst]  # internal target substitutions
     lineFootnoteNum: seq[int]     # footnote line, auto numbers .. [#]
     lineFootnoteNumRef: seq[int]  # footnote line, their reference [#]_
     lineFootnoteSym: seq[int]     # footnote line, auto symbols .. [*]
@@ -508,19 +508,19 @@ type
                                   # number, order of occurrence
     msgHandler: MsgHandler      # How to handle errors.
     findFile: FindFileHandler   # How to find files.
+    filename: string
+    hasToc*: bool
 
-  PSharedState = ref SharedState
+  PRstSharedState* = ref RstSharedState
   RstParser = object of RootObj
     idx*: int
     tok*: TokenSeq
-    s*: PSharedState
+    s*: PRstSharedState
     indentStack*: seq[int]
-    filename*: string
     line*, col*: int            ## initial line/column of whole text or
                                 ## documenation fragment that will be added
                                 ## in case of error/warning reporting to
                                 ## (relative) line/column of the token.
-    hasToc*: bool
     curAnchor*: string          # variable to track latest anchor in s.anchors
 
   EParseError* = object of ValueError
@@ -578,35 +578,42 @@ proc whichRoleAux(sym: string): RstNodeKind =
   else:  # unknown role
     result = rnUnknownRole
 
-proc newSharedState(options: RstParseOptions,
-                    findFile: FindFileHandler,
-                    msgHandler: MsgHandler): PSharedState =
-  new(result)
-  result.currRole = defaultRole(options)
-  result.currRoleKind = whichRoleAux(result.currRole)
-  result.subs = @[]
-  result.refs = @[]
-  result.options = options
-  result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler
-  result.findFile = if not isNil(findFile): findFile else: defaultFindFile
+proc newRstSharedState*(options: RstParseOptions,
+                        filename: string,
+                        findFile: FindFileHandler,
+                        msgHandler: MsgHandler): PRstSharedState =
+  let r = defaultRole(options)
+  result = PRstSharedState(
+      currRole: r,
+      currRoleKind: whichRoleAux(r),
+      options: options,
+      msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler,
+      filename: filename,
+      findFile: if not isNil(findFile): findFile else: defaultFindFile
+  )
 
 proc curLine(p: RstParser): int = p.line + currentTok(p).line
 
 proc findRelativeFile(p: RstParser; filename: string): string =
-  result = p.filename.splitFile.dir / filename
+  result = p.s.filename.splitFile.dir / filename
   if not fileExists(result):
     result = p.s.findFile(filename)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) =
-  p.s.msgHandler(p.filename, curLine(p),
+  p.s.msgHandler(p.s.filename, curLine(p),
                              p.col + currentTok(p).col, msgKind, arg)
 
+proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) =
+  ## Print warnings for footnotes/substitutions.
+  ## TODO: their line/column info is not known, to fix it.
+  s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg)
+
 proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) =
-  p.s.msgHandler(p.filename, p.line + line,
+  p.s.msgHandler(p.s.filename, p.line + line,
                              p.col + col, msgKind, arg)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind) =
-  p.s.msgHandler(p.filename, curLine(p),
+  p.s.msgHandler(p.s.filename, curLine(p),
                              p.col + currentTok(p).col, msgKind,
                              currentTok(p).symbol)
 
@@ -680,12 +687,10 @@ proc popInd(p: var RstParser) =
 # whether it should continue its processing or not, and decided not to,
 # then this B.E. handler should step back (e.g. do `dec p.idx`).
 
-proc initParser(p: var RstParser, sharedState: PSharedState) =
+proc initParser(p: var RstParser, sharedState: PRstSharedState) =
   p.indentStack = @[0]
   p.tok = @[]
   p.idx = 0
-  p.filename = ""
-  p.hasToc = false
   p.col = ColRstInit
   p.line = LineRstInit
   p.s = sharedState
@@ -754,14 +759,14 @@ proc rstnodeToRefname(n: PRstNode): string =
   var b = false
   rstnodeToRefnameAux(n, result, b)
 
-proc findSub(p: var RstParser, n: PRstNode): int =
+proc findSub(s: PRstSharedState, n: PRstNode): int =
   var key = addNodes(n)
   # the spec says: if no exact match, try one without case distinction:
-  for i in countup(0, high(p.s.subs)):
-    if key == p.s.subs[i].key:
+  for i in countup(0, high(s.subs)):
+    if key == s.subs[i].key:
       return i
-  for i in countup(0, high(p.s.subs)):
-    if cmpIgnoreStyle(key, p.s.subs[i].key) == 0:
+  for i in countup(0, high(s.subs)):
+    if cmpIgnoreStyle(key, s.subs[i].key) == 0:
       return i
   result = -1
 
@@ -783,10 +788,10 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) =
       return
   p.s.refs.add(Substitution(key: key, value: value))
 
-proc findRef(p: var RstParser, key: string): PRstNode =
-  for i in countup(0, high(p.s.refs)):
-    if key == p.s.refs[i].key:
-      return p.s.refs[i].value
+proc findRef(s: PRstSharedState, key: string): PRstNode =
+  for i in countup(0, high(s.refs)):
+    if key == s.refs[i].key:
+      return s.refs[i].value
 
 proc addAnchor(p: var RstParser, refn: string, reset: bool) =
   ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor``
@@ -800,8 +805,8 @@ proc addAnchor(p: var RstParser, refn: string, reset: bool) =
   else:
     p.curAnchor = refn
 
-proc findMainAnchor(p: RstParser, refn: string): string =
-  for subst in p.s.anchors:
+proc findMainAnchor(s: PRstSharedState, refn: string): string =
+  for subst in s.anchors:
     if subst.mainAnchor == refn:  # no need to rename
       result = subst.mainAnchor
       break
@@ -838,28 +843,28 @@ proc addFootnoteSymAuto(p: var RstParser) =
   p.s.lineFootnoteSym.add curLine(p)
   p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, ""))
 
-proc orderFootnotes(p: var RstParser) =
+proc orderFootnotes(s: PRstSharedState) =
   ## numerate auto-numbered footnotes taking into account that all
   ## manually numbered ones always have preference.
-  ## Save the result back to p.s.footnotes.
+  ## Save the result back to `s.footnotes`.
 
   # Report an error if found any mismatch in number of automatic footnotes
   proc listFootnotes(lines: seq[int]): string =
     result.add $lines.len & " (lines " & join(lines, ", ") & ")"
-  if p.s.lineFootnoteNum.len != p.s.lineFootnoteNumRef.len:
-    rstMessage(p, meFootnoteMismatch,
-      "$1 != $2" % [listFootnotes(p.s.lineFootnoteNum),
-                    listFootnotes(p.s.lineFootnoteNumRef)] &
+  if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len:
+    rstMessage(s, meFootnoteMismatch,
+      "$1 != $2" % [listFootnotes(s.lineFootnoteNum),
+                    listFootnotes(s.lineFootnoteNumRef)] &
         " for auto-numbered footnotes")
-  if p.s.lineFootnoteSym.len != p.s.lineFootnoteSymRef.len:
-    rstMessage(p, meFootnoteMismatch,
-      "$1 != $2" % [listFootnotes(p.s.lineFootnoteSym),
-                    listFootnotes(p.s.lineFootnoteSymRef)] &
+  if s.lineFootnoteSym.len != s.lineFootnoteSymRef.len:
+    rstMessage(s, meFootnoteMismatch,
+      "$1 != $2" % [listFootnotes(s.lineFootnoteSym),
+                    listFootnotes(s.lineFootnoteSymRef)] &
         " for auto-symbol footnotes")
 
   var result: seq[FootnoteSubst]
   var manuallyN, autoN, autoSymbol: seq[FootnoteSubst]
-  for fs in p.s.footnotes:
+  for fs in s.footnotes:
     if fs.kind == fnManualNumber: manuallyN.add fs
     elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs
     else: autoSymbol.add fs
@@ -906,26 +911,26 @@ proc orderFootnotes(p: var RstParser) =
     let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols)
     result.add((fs.kind, -1, -1, fs.autoSymIdx, label))
 
-  p.s.footnotes = result
+  s.footnotes = result
 
-proc getFootnoteNum(p: var RstParser, label: string): int =
+proc getFootnoteNum(s: PRstSharedState, label: string): int =
   ## get number from label. Must be called after `orderFootnotes`.
   result = -1
-  for fnote in p.s.footnotes:
+  for fnote in s.footnotes:
     if fnote.label == label:
       return fnote.number
 
-proc getFootnoteNum(p: var RstParser, order: int): int =
+proc getFootnoteNum(s: PRstSharedState, order: int): int =
   ## get number from occurrence. Must be called after `orderFootnotes`.
   result = -1
-  for fnote in p.s.footnotes:
+  for fnote in s.footnotes:
     if fnote.autoNumIdx == order:
       return fnote.number
 
-proc getAutoSymbol(p: var RstParser, order: int): string =
+proc getAutoSymbol(s: PRstSharedState, order: int): string =
   ## get symbol from occurrence of auto-symbol footnote.
   result = "???"
-  for fnote in p.s.footnotes:
+  for fnote in s.footnotes:
     if fnote.autoSymIdx == order:
       return fnote.label
 
@@ -1765,17 +1770,18 @@ proc getLevel(p: var RstParser, c: char, hasOverline: bool): int =
                             line: curLine(p), hasPeers: false)
   result = p.s.hLevels.len - 1
 
-proc countTitles(p: var RstParser, n: PRstNode) =
-  ## Fill `p.s.hTitleCnt`
+proc countTitles(s: PRstSharedState, n: PRstNode) =
+  ## Fill `s.hTitleCnt`
+  if n == nil: return
   for node in n.sons:
     if node != nil:
       if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}:
         break
       if node.kind == rnOverline:
-        if p.s.hLevels[p.s.hTitleCnt].hasPeers:
+        if s.hLevels[s.hTitleCnt].hasPeers:
           break
-        inc p.s.hTitleCnt
-        if p.s.hTitleCnt >= 2:
+        inc s.hTitleCnt
+        if s.hTitleCnt >= 2:
           break
 
 proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool =
@@ -2103,8 +2109,7 @@ proc parseSimpleTable(p: var RstParser): PRstNode =
       initParser(q, p.s)
       q.col = cols[j]
       q.line = line - 1
-      q.filename = p.filename
-      q.col += getTokens(row[j], q.tok)
+      getTokens(row[j], q.tok)
       b = newRstNode(rnTableDataCell)
       b.add(parseDoc(q))
       a.add(b)
@@ -2156,8 +2161,7 @@ proc parseMarkdownTable(p: var RstParser): PRstNode =
       initParser(q, p.s)
       q.col = p.col
       q.line = currentTok(p).line - 1
-      q.filename = p.filename
-      q.col += getTokens(getColContents(p, row[j]), q.tok)
+      getTokens(getColContents(p, row[j]), q.tok)
       b.add(parseDoc(q))
       a.add(b)
     result.add(a)
@@ -2562,8 +2566,8 @@ proc dirInclude(p: var RstParser): PRstNode =
 
       var q: RstParser
       initParser(q, p.s)
-      q.filename = path
-      q.col += getTokens(
+      q.s.filename = path
+      getTokens(
         inputString[startPosition..endPosition].strip(),
         q.tok)
       # workaround a GCC bug; more like the interior pointer bug?
@@ -2806,7 +2810,27 @@ proc parseDotDot(p: var RstParser): PRstNode =
   else:
     result = parseComment(p)
 
-proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
+proc rstParsePass1*(fragment, filename: string,
+                    line, column: int,
+                    options: RstParseOptions,
+                    sharedState: PRstSharedState): PRstNode =
+  ## Parses an RST `fragment`.
+  ## The result should be further processed by
+  ## `preparePass2` and `resolveSubs` (which is pass 2).
+  var p: RstParser
+  initParser(p, sharedState)
+  p.line = line
+  p.col = column
+  getTokens(fragment, p.tok)
+  result = parseDoc(p)
+
+proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) =
+  ## Records titles in node `mainNode` and orders footnotes.
+  countTitles(s, mainNode)
+  orderFootnotes(s)
+
+proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
+  ## Makes pass 2 of RST parsing.
   ## Resolves substitutions and anchor aliases, groups footnotes.
   ## Takes input node `n` and returns the same node with recursive
   ## substitutions in `n.sons` to `result`.
@@ -2814,32 +2838,32 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
   if n == nil: return
   case n.kind
   of rnSubstitutionReferences:
-    var x = findSub(p, n)
+    var x = findSub(s, n)
     if x >= 0:
-      result = p.s.subs[x].value
+      result = s.subs[x].value
     else:
       var key = addNodes(n)
       var e = getEnv(key)
       if e != "": result = newLeaf(e)
-      else: rstMessage(p, mwUnknownSubstitution, key)
+      else: rstMessage(s, mwUnknownSubstitution, key)
   of rnHeadline, rnOverline:
     # fix up section levels depending on presence of a title and subtitle
-    if p.s.hTitleCnt == 2:
+    if s.hTitleCnt == 2:
       if n.level == 1:    # it's the subtitle
         n.level = 0
       elif n.level >= 2:  # normal sections
         n.level -= 1
-    elif p.s.hTitleCnt == 0:
+    elif s.hTitleCnt == 0:
       n.level += 1
   of rnRef:
     let refn = rstnodeToRefname(n)
-    var y = findRef(p, refn)
+    var y = findRef(s, refn)
     if y != nil:
       result = newRstNode(rnHyperlink)
       let text = newRstNode(rnInner, n.sons)
       result.sons = @[text, y]
     else:
-      let s = findMainAnchor(p, refn)
+      let s = findMainAnchor(s, refn)
       if s != "":
         result = newRstNode(rnInternalRef)
         let text = newRstNode(rnInner, n.sons)
@@ -2853,16 +2877,16 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
     of fnAutoNumberLabel, fnAutoNumber:
       if fnType == fnAutoNumberLabel:
         let labelR = rstnodeToRefname(n.sons[0])
-        num = getFootnoteNum(p, labelR)
+        num = getFootnoteNum(s, labelR)
       else:
-        num = getFootnoteNum(p, n.order)
+        num = getFootnoteNum(s, n.order)
       var nn = newRstNode(rnInner)
       nn.add newLeaf($num)
       result.sons[0] = nn
     of fnAutoSymbol:
-      let sym = getAutoSymbol(p, n.order)
+      let sym = getAutoSymbol(s, n.order)
       n.sons[0].sons[0].text = sym
-    n.sons[1] = resolveSubs(p, n.sons[1])
+    n.sons[1] = resolveSubs(s, n.sons[1])
   of rnFootnoteRef:
     var (fnType, num) = getFootnoteType(n.sons[0])
     template addLabel(number: int | string) =
@@ -2877,31 +2901,31 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
       addLabel num
       refn.add $num
     of fnAutoNumber:
-      addLabel getFootnoteNum(p, n.order)
+      addLabel getFootnoteNum(s, n.order)
       refn.add $n.order
     of fnAutoNumberLabel:
-      addLabel getFootnoteNum(p, rstnodeToRefname(n))
+      addLabel getFootnoteNum(s, rstnodeToRefname(n))
       refn.add rstnodeToRefname(n)
     of fnAutoSymbol:
-      addLabel getAutoSymbol(p, n.order)
+      addLabel getAutoSymbol(s, n.order)
       refn.add $n.order
     of fnCitation:
       result.add n.sons[0]
       refn.add rstnodeToRefname(n)
-    let s = findMainAnchor(p, refn)
-    if s != "":
-      result.add newLeaf(s)     # add link
+    let anch = findMainAnchor(s, refn)
+    if anch != "":
+      result.add newLeaf(anch)     # add link
     else:
-      rstMessage(p, mwUnknownSubstitution, refn)
+      rstMessage(s, mwUnknownSubstitution, refn)
       result.add newLeaf(refn)  # add link
   of rnLeaf:
     discard
   of rnContents:
-    p.hasToc = true
+    s.hasToc = true
   else:
     var regroup = false
     for i in 0 ..< n.len:
-      n.sons[i] = resolveSubs(p, n.sons[i])
+      n.sons[i] = resolveSubs(s, n.sons[i])
       if n.sons[i] != nil and n.sons[i].kind == rnFootnote:
         regroup = true
     if regroup:  # group footnotes together into rnFootnoteGroup
@@ -2924,13 +2948,10 @@ proc rstParse*(text, filename: string,
                options: RstParseOptions,
                findFile: FindFileHandler = nil,
                msgHandler: MsgHandler = nil): PRstNode =
-  var p: RstParser
-  initParser(p, newSharedState(options, findFile, msgHandler))
-  p.filename = filename
-  p.line = line
-  p.col = column + getTokens(text, p.tok)
-  let unresolved = parseDoc(p)
-  countTitles(p, unresolved)
-  orderFootnotes(p)
-  result = resolveSubs(p, unresolved)
-  hasToc = p.hasToc
+  ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`.
+  var sharedState = newRstSharedState(options, filename, findFile, msgHandler)
+  let unresolved = rstParsePass1(text, filename, line, column,
+                                 options, sharedState)
+  preparePass2(sharedState, unresolved)
+  result = resolveSubs(sharedState, unresolved)
+  hasToc = sharedState.hasToc

nimdoc/testproject/expected/subdir/subdir_b/utils.idx

@@ -1,8 +1,3 @@
-This is now a header	subdir/subdir_b/utils.html#this-is-now-a-header	 This is now a header	
-Next header	subdir/subdir_b/utils.html#this-is-now-a-header-next-header	  Next header	
-And so on	subdir/subdir_b/utils.html#next-header-and-so-on	   And so on	
-More headers	subdir/subdir_b/utils.html#more-headers	 More headers	
-Up to level 6	subdir/subdir_b/utils.html#more-headers-up-to-level-6	      Up to level 6	
 enumValueA	subdir/subdir_b/utils.html#enumValueA	SomeType.enumValueA	
 enumValueB	subdir/subdir_b/utils.html#enumValueB	SomeType.enumValueB	
 enumValueC	subdir/subdir_b/utils.html#enumValueC	SomeType.enumValueC	
@@ -11,3 +6,8 @@ someType	subdir/subdir_b/utils.html#someType_2	utils: someType(): SomeType
 aEnum	subdir/subdir_b/utils.html#aEnum.t	utils: aEnum(): untyped	
 bEnum	subdir/subdir_b/utils.html#bEnum.t	utils: bEnum(): untyped	
 fromUtilsGen	subdir/subdir_b/utils.html#fromUtilsGen.t	utils: fromUtilsGen(): untyped	
+This is now a header	subdir/subdir_b/utils.html#this-is-now-a-header	 This is now a header	
+Next header	subdir/subdir_b/utils.html#this-is-now-a-header-next-header	  Next header	
+And so on	subdir/subdir_b/utils.html#next-header-and-so-on	   And so on	
+More headers	subdir/subdir_b/utils.html#more-headers	 More headers	
+Up to level 6	subdir/subdir_b/utils.html#more-headers-up-to-level-6	      Up to level 6	

tests/stdlib/trstgen.nim

@@ -997,7 +997,7 @@ Test1
       """
     var error5 = new string
     let output5 = input5.toHtml(error=error5)
-    check(error5[] == "input(6, 1) Error: mismatch in number of footnotes " &
+    check(error5[] == "input(1, 1) Error: mismatch in number of footnotes " &
             "and their refs: 1 (lines 2) != 0 (lines ) for auto-numbered " &
             "footnotes")
 
@@ -1011,7 +1011,7 @@ Test1
       """
     var error6 = new string
     let output6 = input6.toHtml(error=error6)
-    check(error6[] == "input(6, 1) Error: mismatch in number of footnotes " &
+    check(error6[] == "input(1, 1) Error: mismatch in number of footnotes " &
             "and their refs: 1 (lines 3) != 2 (lines 2, 6) for auto-symbol " &
             "footnotes")
 
@@ -1034,7 +1034,8 @@ Test1
       """
     var warnings8 = new seq[string]
     let output8 = input8.toHtml(warnings=warnings8)
-    check(warnings8[] == @["input(4, 1) Warning: unknown substitution " &
+    # TODO: the line 1 is arbitrary because reference lines are not preserved
+    check(warnings8[] == @["input(1, 1) Warning: unknown substitution " &
             "\'citation-som\'"])
 
     # check that footnote group does not break parsing of other directives: