fix #6583, fix #14376, index+search now generated for all projects, many bug fixes with nim doc (#14324)

* refs #6583 fix nim doc output * changelog * change default for outDir when unspecified * cleanups * --project implies --index
2025-12-28 08:54:53 +00:00 · 2020-05-25 04:25:40 -07:00
parent cbfe9325c5
commit 58282547f6
22 changed files with 294 additions and 161 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -92,7 +92,8 @@ megatest.nim
 # ignore debug dirs generated by dsymutil on OSX
 *.dSYM

-nimdoc.out.css
-
 # for `nim c -r nimdoc/tester` etc; this can be in multiple places
 htmldocs
+
+## these are not needed anymore unless checkout old older versions
+nimdoc.out.css
--- a/changelog.md
+++ b/changelog.md
@@ -94,7 +94,6 @@
 - The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
 - The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.

- `osproc.execCmdEx` now takes an optional `input` for stdin.
 - `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
  parameters.

@@ -188,6 +187,9 @@ proc mydiv(a, b): int {.raises: [].} =
  and avoids polluting both $pwd and $projectdir. It can be used with any command.
 - `runnableExamples "-b:cpp -r:off": code` is now supported, allowing to override how an example is compiled and run,
  for example to change backend or compile only.
+- `nim doc` now outputs under `$projectPath/htmldocs` when `--outdir` is unspecified (with or without `--project`);
+  passing `--project` now automatically generates an index and enables search.
+  See [docgen](docgen.html#introduction-quick-start) for details.

 ## Tool changes

--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -440,7 +440,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
    expectArg(conf, switch, arg, pass, info)
    conf.docSeeSrcUrl = arg
  of "docroot":
-    conf.docRoot = if arg.len == 0: "@default" else: arg
+    conf.docRoot = if arg.len == 0: docRootDefault else: arg
  of "backend", "b":
    let backend = parseEnum(arg.normalize, TBackend.default)
    if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
@@ -485,7 +485,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
  of "forcebuild", "f":
    processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
  of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
  of "gc":
    expectArg(conf, switch, arg, pass, info)
    if pass in {passCmd2, passPP}:
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -17,11 +17,10 @@ import
  packages/docutils/rst, packages/docutils/rstgen,
  json, xmltree, cgi, trees, types,
  typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees, tables
+  pathutils, trees, tables, nimpaths

 const
  exportSection = skField
-  htmldocsDir = RelativeDir"htmldocs"
  docCmdSkip = "skip"

 type
@@ -51,7 +50,7 @@ type
    destFile*: AbsoluteFile
    thisDir*: AbsoluteDir
    exampleGroups: OrderedTable[string, ExampleGroup]
-    wroteCss*: bool
+    wroteSupportFiles*: bool

  PDoc* = ref TDocumentor ## Alias to type less.

@@ -72,7 +71,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
  proc nimbleDir(): AbsoluteDir =
    getNimbleFile(conf, file2).parentDir.AbsoluteDir
  case conf.docRoot:
-  of "@default": # using `@` instead of `$` to avoid shell quoting complications
+  of docRootDefault:
    result = getRelativePathFromConfigPath(conf, file)
    let dir = nimbleDir()
    if not dir.isEmpty:
@@ -88,9 +87,12 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
    result = getRelativePathFromConfigPath(conf, file)
    if result.isEmpty: bail()
  elif conf.docRoot.len > 0:
-    doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError
-    doAssert conf.docRoot.existsDir, conf.docRoot
-    result = relativeTo(file, conf.docRoot.AbsoluteDir)
+    # we're (currently) requiring `isAbsolute` to avoid confusion when passing
+    # a relative path (would it be relative wrt $PWD or to projectfile)
+    conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot
+    conf.globalAssert conf.docRoot.existsDir, arg=conf.docRoot
+    # needed because `canonicalizePath` called on `file`
+    result = file.relativeTo conf.docRoot.expandFilename.AbsoluteDir
  else:
    bail()
  if isAbsolute(result.string):
@@ -1125,11 +1127,8 @@ proc genSection(d: PDoc, kind: TSymKind) =
      "sectionid", "sectionTitle", "sectionTitleID", "content"], [
      ord(kind).rope, title, rope(ord(kind) + 50), d.toc[kind]])

-const nimdocOutCss = "nimdoc.out.css"
-  # `out` to make it easier to use with gitignore in user's repos
-
-proc cssHref(outDir: AbsoluteDir, destFile: AbsoluteFile): Rope =
-  rope($relativeTo(outDir / nimdocOutCss.RelativeFile, destFile.splitFile().dir, '/'))
+proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): Rope =
+  rope($relativeTo(outDir / linkto, destFile.splitFile().dir, '/'))

 proc genOutFile(d: PDoc): Rope =
  var
@@ -1160,15 +1159,17 @@ proc genOutFile(d: PDoc): Rope =
                 elif d.hasToc: "doc.body_toc"
                 else: "doc.body_no_toc"
  content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title",
-      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg"],
+      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg", "theindexhref"],
      [title.rope, toc, d.modDesc, rope(getDateStr()),
-      rope(getClockStr()), code, d.modDeprecationMsg])
+      rope(getClockStr()), code, d.modDeprecationMsg, relLink(d.conf.outDir, d.destFile, theindexFname.RelativeFile)])
  if optCompileOnly notin d.conf.globalOptions:
    # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
    code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), [
-        "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+        "nimdoccss", "dochackjs",  "title", "tableofcontents", "moduledesc", "date", "time",
        "content", "author", "version", "analytics", "deprecationMsg"],
-        [cssHref(d.conf.outDir, d.destFile), title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
+        [relLink(d.conf.outDir, d.destFile, nimdocOutCss.RelativeFile),
+        relLink(d.conf.outDir, d.destFile, docHackJsFname.RelativeFile),
+        title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
        content, d.meta[metaAuthor].rope, d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
  else:
    code = content
@@ -1203,11 +1204,13 @@ proc writeOutput*(d: PDoc, useWarning = false) =
    if not writeRope(content, outfile):
      rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
        outfile.string)
-    elif not d.wroteCss:
-      let cssSource = $d.conf.getPrefixDir() / "doc" / "nimdoc.css"
-      let cssDest = $dir / nimdocOutCss
-      copyFile(cssSource, cssDest)
-      d.wroteCss = true
+    elif not d.wroteSupportFiles: # nimdoc.css + dochack.js
+      let nimr = $d.conf.getPrefixDir()
+      copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss)
+      if optGenIndex in d.conf.globalOptions:
+        let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename())
+        copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart)
+      d.wroteSupportFiles = true

 proc writeOutputJson*(d: PDoc, useWarning = false) =
  runAllExamples(d)
@@ -1234,6 +1237,8 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
 proc handleDocOutputOptions*(conf: ConfigRef) =
  if optWholeProject in conf.globalOptions:
    # Backward compatibility with previous versions
+    # xxx this is buggy when user provides `nim doc --project -o:sub/bar.html main`,
+    # it'd write to `sub/bar.html/main.html`
    conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)

 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
@@ -1308,20 +1313,23 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
    if not writeRope(content, filename):
      rawMessage(conf, errCannotOpenFile, filename.string)

-proc commandBuildIndex*(cache: IdentCache, conf: ConfigRef) =
-  var content = mergeIndexes(conf.projectFull.string).rope
+proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
+  var content = mergeIndexes(dir).rope

-  var outFile = RelativeFile"theindex"
-  if conf.outFile != RelativeFile"":
-    outFile = conf.outFile
+  var outFile = outFile
+  if outFile.isEmpty: outFile = theindexFname.RelativeFile.changeFileExt("")
  let filename = getOutFile(conf, outFile, HtmlExt)

  let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), [
-      "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+      "nimdoccss", "dochackjs",
+      "title", "tableofcontents", "moduledesc", "date", "time",
      "content", "author", "version", "analytics"],
-      [cssHref(conf.outDir, filename), rope"Index", nil, nil, rope(getDateStr()),
+      [relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
+      relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
+      rope"Index", nil, nil, rope(getDateStr()),
      rope(getClockStr()), content, nil, nil, nil])
  # no analytics because context is not available

  if not writeRope(code, filename):
    rawMessage(conf, errCannotOpenFile, filename.string)
+
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -66,16 +66,8 @@ when not defined(leanCompiler):
    compileProject(graph)
    finishDoc2Pass(graph.config.projectName)

-proc setOutDir(conf: ConfigRef) =
-  if conf.outDir.isEmpty:
-    if optUseNimcache in conf.globalOptions:
-      conf.outDir = getNimcacheDir(conf)
-    else:
-      conf.outDir = conf.projectPath
-
 proc commandCompileToC(graph: ModuleGraph) =
  let conf = graph.config
-  setOutDir(conf)
  if conf.outFile.isEmpty:
    let base = conf.projectName
    let targetName = if optGenDynLib in conf.globalOptions:
@@ -121,7 +113,6 @@ proc commandCompileToJS(graph: ModuleGraph) =
    let conf = graph.config
    conf.exc = excCpp

-    setOutDir(conf)
    if conf.outFile.isEmpty:
      conf.outFile = RelativeFile(conf.projectName & ".js")

@@ -191,11 +182,6 @@ proc mainCommand*(graph: ModuleGraph) =
  conf.searchPaths.add(conf.libpath)
  setId(100)

-  ## Calling `setOutDir(conf)` unconditionally would fix regression
-  ## https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
-  when false: setOutDir(conf)
-  if optUseNimcache in conf.globalOptions: setOutDir(conf)
-
  proc customizeForBackend(backend: TBackend) =
    ## Sets backend specific options but don't compile to backend yet in
    ## case command doesn't require it. This must be called by all commands.
@@ -234,15 +220,40 @@ proc mainCommand*(graph: ModuleGraph) =
    of backendJs: commandCompileToJS(graph)
    of backendInvalid: doAssert false

+  template docLikeCmd(body) =
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      wantMainModule(conf)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      body
+
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+        else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
+
  ## process all backend commands
  case conf.command.normalize
  of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
  of "cpp", "compiletocpp": compileToBackend(backendCpp)
  of "objc", "compiletooc": compileToBackend(backendObjc)
  of "js", "compiletojs": compileToBackend(backendJs)
-  of "r": # different from `"run"`!
-    conf.globalOptions.incl {optRun, optUseNimcache}
-    compileToBackend(backendC)
+  of "r": compileToBackend(backendC) # different from `"run"`!
  of "run":
    when hasTinyCBackend:
      extccomp.setCC(conf, "tcc", unknownLineInfo)
@@ -254,29 +265,19 @@ proc mainCommand*(graph: ModuleGraph) =
  else: customizeForBackend(backendC) # fallback for other commands

  ## process all other commands
-  case conf.command.normalize
-  of "doc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
+  of "doc0": docLikeCmd commandDoc(cache, conf)
  of "doc2", "doc":
-    conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
-      # because currently generates lots of false positives due to conflation
-      # of labels links in doc comments, eg for random.rand:
-      #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
-      #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
+    docLikeCmd():
+      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+        # because currently generates lots of false positives due to conflation
+        # of labels links in doc comments, eg for random.rand:
+        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
      commandDoc2(graph, false)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndex(conf, $conf.outDir)
  of "rst2html":
    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
    when defined(leanCompiler):
@@ -292,41 +293,10 @@ proc mainCommand*(graph: ModuleGraph) =
      conf.cmd = cmdRst2tex
      loadConfigs(DocTexConfig, cache, conf)
      commandRst2TeX(cache, conf)
-  of "jsondoc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
-  of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
-  of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
-  of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
+  of "ctags": docLikeCmd commandTags(cache, conf)
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
  of "gendepend":
    conf.cmd = cmdGenDepend
    commandGenDepend(graph)
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -510,7 +510,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
        styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg)
        if conf.hasHint(hintSource) and info != unknownLineInfo:
          conf.writeSurroundingSrc(info)
-        if conf.hasHint(hintMsgOrigin):
+        if hintMsgOrigin in conf.mainPackageNotes:
          styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
            " compiler msg initiated here", KindColor,
            KindFormat % hintMsgOrigin.msgToStr,
@@ -529,6 +529,14 @@ template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
  conf.m.errorOutputs = {eStdOut, eStdErr}
  liMessage(conf, info, msg, arg, doAbort, instLoc())

+template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") =
+  ## avoids boilerplate
+  if not cond:
+    const info2 = instantiationInfo(-1, fullPaths = true)
+    var arg2 = "'$1' failed" % [astToStr(cond)]
+    if arg.len > 0: arg2.add "; " & astToStr(arg) & ": " & arg
+    liMessage(conf, info, errGenerated, arg2, doRaise, info2)
+
 template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
  liMessage(conf, info, msg, arg, doRaise, instLoc())

--- a/compiler/nimpaths.nim
+++ b/compiler/nimpaths.nim
@@ -0,0 +1,44 @@
+##[
+Represents absolute paths, but using a symbolic variables (eg $nimr) which can be
+resolved at runtime; this avoids hardcoding at compile time absolute paths so
+that the project root can be relocated.
+
+xxx consider some refactoring with $nim/testament/lib/stdtest/specialpaths.nim;
+specialpaths is simpler because it doesn't need variables to be relocatable at
+runtime (eg for use in testament)
+
+interpolation variables:
+  $nimr: such that `$nimr/lib/system.nim` exists (avoids confusion with $nim binary)
+         in compiler, it's obtainable via getPrefixDir(); for other tools (eg koch),
+        this could be getCurrentDir() or getAppFilename().parentDir.parentDir,
+        depending on use case
+
+Unstable API
+]##
+
+import std/[os,strutils]
+
+const
+  docCss* = "$nimr/doc/nimdoc.css"
+  docHackNim* = "$nimr/tools/dochack/dochack.nim"
+  docHackJs* = docHackNim.changeFileExt("js")
+  docHackJsFname* = docHackJs.lastPathPart
+  theindexFname* = "theindex.html"
+  nimdocOutCss* = "nimdoc.out.css"
+    # `out` to make it easier to use with gitignore in user's repos
+  htmldocsDirname* = "htmldocs"
+
+proc interp*(path: string, nimr: string): string =
+  result = path % ["nimr", nimr]
+  doAssert '$' notin result, $(path, nimr, result) # avoids un-interpolated variables in output
+
+proc getDocHacksJs*(nimr: string, nim = getCurrentCompilerExe(), forceRebuild = false): string =
+  ## return absolute path to dochhack.js, rebuilding if it doesn't exist or if
+  ## `forceRebuild`.
+  let docHackJs2 = docHackJs.interp(nimr = nimr)
+  if forceRebuild or not docHackJs2.fileExists:
+    let cmd =  "$nim js $file" % ["nim", nim.quoteShell, "file", docHackNim.interp(nimr = nimr).quoteShell]
+    echo "getDocHacksJs: cmd: " & cmd
+    doAssert execShellCmd(cmd) == 0, $(cmd)
+  doAssert docHackJs2.fileExists
+  result = docHackJs2
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -9,7 +9,7 @@

 import
  os, strutils, strtabs, sets, lineinfos, platform,
-  prefixmatches, pathutils
+  prefixmatches, pathutils, nimpaths

 from terminal import isatty
 from times import utc, fromUnix, local, getTime, format, DateTime
@@ -518,8 +518,9 @@ const
  DefaultConfigNims* = RelativeFile"config.nims"
  DocConfig* = RelativeFile"nimdoc.cfg"
  DocTexConfig* = RelativeFile"nimdoc.tex.cfg"
-
-const oKeepVariableNames* = true
+  htmldocsDir* = htmldocsDirname.RelativeDir
+  docRootDefault* = "@default" # using `@` instead of `$` to avoid shell quoting complications
+  oKeepVariableNames* = true

 proc mainCommandArg*(conf: ConfigRef): string =
  ## This is intended for commands like check or parse
@@ -543,6 +544,7 @@ proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): Absolute
  # explains regression https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
  # Yet another reason why "" should not mean ".";  `""/something` should raise
  # instead of implying "" == "." as it's bug prone.
+  doAssert conf.outDir.string.len > 0
  result = conf.outDir / changeFileExt(filename, ext)

 proc absOutFile*(conf: ConfigRef): AbsoluteFile =
--- a/config/nimdoc.cfg
+++ b/config/nimdoc.cfg
@@ -100,7 +100,7 @@ doc.body_toc = """
        <a href="lib.html">Standard library</a>
      </li>
      <li>
-        <a href="theindex.html">Index</a>
+        <a href="$theindexhref">Index</a>
      </li>
    </ul>
  </div>
@@ -143,7 +143,7 @@ doc.body_toc_group = """
        <a href="lib.html">Standard library</a>
      </li>
      <li>
-        <a href="theindex.html">Index</a>
+        <a href="$theindexhref">Index</a>
      </li>
    </ul>
  </div>
@@ -183,6 +183,9 @@ doc.body_toc_group = """
  </div>
  <div id="global-links">
    <ul class="simple">
+    <li>
+      <a href="$theindexhref">Index</a>
+    </li>
    </ul>
  </div>
  <div id="searchInputDiv">
@@ -239,7 +242,7 @@ doc.file = """<?xml version="1.0" encoding="utf-8" ?>
 <title>$title</title>
 <link rel="stylesheet" type="text/css" href="$nimdoccss">

-<script type="text/javascript" src="dochack.js"></script>
+<script type="text/javascript" src="$dochackjs"></script>

 <script type="text/javascript">
 function main() {
--- a/doc/docgen.rst
+++ b/doc/docgen.rst
@@ -30,8 +30,16 @@ Generate HTML documentation for a whole project:

 ::
  # delete any htmldocs/*.idx file before starting
-  nim doc --project --index:on --git.url:<url> --git.commit:<tag> <main_filename>.nim
-  nim buildIndex -o:htmldocs/theindex.html htmldocs
+  nim doc --project --index:on --git.url:<url> --git.commit:<tag> --outdir:htmldocs <main_filename>.nim
+  # this will generate html files, a theindex.html index, css and js under `htmldocs`
+  # See also `--docroot` to specify a relative root.
+  # to get search (dochacks.js) to work locally, you need a server otherwise
+  # CORS will prevent opening file:// urls; this works:
+  python3 -m http.server 7029 --directory htmldocs
+  # When --outdir is omitted it defaults to $projectPath/htmldocs,
+  or `$nimcache/htmldocs` with `--usenimcache` which avoids clobbering your sources;
+  and likewise without `--project`.
+  Adding `-r` will open in a browser directly.


 Documentation Comments
--- a/nimdoc/test_out_index_dot_html/expected/index.html
+++ b/nimdoc/test_out_index_dot_html/expected/index.html
@@ -83,6 +83,9 @@ function main() {
  </div>
  <div id="global-links">
    <ul class="simple">
+    <li>
+      <a href="theindex.html">Index</a>
+    </li>
    </ul>
  </div>
  <div id="searchInputDiv">
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -20,7 +20,7 @@
 <title>subdir/subdir_b/utils</title>
 <link rel="stylesheet" type="text/css" href="../../nimdoc.out.css">

-<script type="text/javascript" src="dochack.js"></script>
+<script type="text/javascript" src="../../dochack.js"></script>

 <script type="text/javascript">
 function main() {
@@ -83,6 +83,9 @@ function main() {
  </div>
  <div id="global-links">
    <ul class="simple">
+    <li>
+      <a href="../../theindex.html">Index</a>
+    </li>
    </ul>
  </div>
  <div id="searchInputDiv">
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -83,6 +83,9 @@ function main() {
  </div>
  <div id="global-links">
    <ul class="simple">
+    <li>
+      <a href="theindex.html">Index</a>
+    </li>
    </ul>
  </div>
  <div id="searchInputDiv">
--- a/testament/lib/stdtest/specialpaths.nim
+++ b/testament/lib/stdtest/specialpaths.nim
@@ -1,5 +1,6 @@
 #[
 todo: move findNimStdLibCompileTime, findNimStdLib here
+xxx: consider moving this to $nim/compiler/relpaths.nim to get relocatable paths
 ]#

 import os
@@ -9,22 +10,17 @@ import os
 # This means the binaries they produce will embed hardcoded paths, which
 # isn't appropriate for some applications that need to be relocatable.

-const sourcePath = currentSourcePath()
-  # robust way to derive other paths here
-  # We don't depend on PATH so this is robust to having multiple nim
-  # binaries
-
-const nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir
-  ## root of Nim repo
-
-const stdlibDir* = nimRootDir / "lib"
-  # todo: make nimeval.findNimStdLibCompileTime use this
-
-const systemPath* = stdlibDir / "system.nim"
-
-const buildDir* = nimRootDir / "build"
-  ## refs #10268: all testament generated files should go here to avoid
-  ## polluting .gitignore
+const
+  sourcePath = currentSourcePath()
+    # robust way to derive other paths here
+    # We don't depend on PATH so this is robust to having multiple nim binaries
+  nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir ## root of Nim repo
+  stdlibDir* = nimRootDir / "lib"
+  systemPath* = stdlibDir / "system.nim"
+  testsDir* = nimRootDir / "tests"
+  buildDir* = nimRootDir / "build"
+    ## refs #10268: all testament generated files should go here to avoid
+    ## polluting .gitignore

 static:
  # sanity check
--- a/testament/lib/stdtest/unittest_light.nim
+++ b/testament/lib/stdtest/unittest_light.nim
@@ -1,5 +1,3 @@
-# note: consider merging tests/assert/testhelper.nim here.
-
 proc mismatch*[T](lhs: T, rhs: T): string =
  ## Simplified version of `unittest.require` that satisfies a common use case,
  ## while avoiding pulling too many dependencies. On failure, diagnostic
@@ -31,8 +29,8 @@ proc mismatch*[T](lhs: T, rhs: T): string =
      result.add "lhs[0..<i]:{" & replaceInvisible($lhs[
          0..<i]) & "}"

-proc assertEquals*[T](lhs: T, rhs: T) =
+proc assertEquals*[T](lhs: T, rhs: T, msg = "") =
  when false: # can be useful for debugging to see all that's fed to this.
    echo "----" & $lhs
  if lhs!=rhs:
-    doAssert false, mismatch(lhs, rhs)
+    doAssert false, mismatch(lhs, rhs) & "\n" & msg
--- a/tests/misc/trunner.nim
+++ b/tests/misc/trunner.nim
@@ -8,19 +8,20 @@ discard """
 ## Note: this test is a bit slow but tests a lot of things; please don't disable.

 import std/[strformat,os,osproc,unittest]
+from std/sequtils import toSeq,mapIt
+from std/algorithm import sorted
+import stdtest/[specialpaths, unittest_light]

-const nim = getCurrentCompilerExe()
+import "$nim/compiler/nimpaths"

-const mode =
-  when defined(c): "c"
-  elif defined(cpp): "cpp"
-  else: static: doAssert false
-
-const testsDir = currentSourcePath.parentDir.parentDir
-const buildDir = testsDir.parentDir / "build"
-const nimcache = buildDir / "nimcacheTrunner"
-  # `querySetting(nimcacheDir)` would also be possible, but we thus
-  # avoid stomping on other parallel tests
+const
+  nim = getCurrentCompilerExe()
+  mode =
+    when defined(c): "c"
+    elif defined(cpp): "cpp"
+    else: static: doAssert false
+  nimcache = buildDir / "nimcacheTrunner"
+    # instead of `querySetting(nimcacheDir)`, avoids stomping on other parallel tests

 proc runCmd(file, options = ""): auto =
  let fileabs = testsDir / file.unixToNativePath
@@ -63,6 +64,82 @@ else: # don't run twice the same test
  import std/[strutils]
  template check2(msg) = doAssert msg in output, output

+  block: # tests with various options `nim doc --project --index --docroot`
+    # regression tests for issues and PRS: #14376 #13223 #6583 ##13647
+    let file = testsDir / "nimdoc/sub/mmain.nim"
+    let mainFname = "mmain.html"
+    let htmldocsDirCustom = nimcache / "htmldocsCustom"
+    let docroot = testsDir / "nimdoc"
+    let options = [
+      0: "--project",
+      1: "--project --docroot",
+      2: "",
+      3: fmt"--outDir:{htmldocsDirCustom}",
+      4: fmt"--docroot:{docroot}",
+      5: "--project --useNimcache",
+      6: "--index:off",
+    ]
+
+    for i in 0..<options.len:
+      let htmldocsDir = case i
+      of 3: htmldocsDirCustom
+      of 5: nimcache / htmldocsDirname
+      else: file.parentDir / htmldocsDirname
+
+      var cmd = fmt"{nim} doc --index:on --listFullPaths --hint:successX:on --nimcache:{nimcache} {options[i]} {file}"
+      removeDir(htmldocsDir)
+      let (outp, exitCode) = execCmdEx(cmd)
+      check exitCode == 0
+      proc nativeToUnixPathWorkaround(a: string): string =
+        # xxx pending https://github.com/nim-lang/Nim/pull/13265 `nativeToUnixPath`
+        a.replace(DirSep, '/')
+
+      let ret = toSeq(walkDirRec(htmldocsDir, relative=true)).mapIt(it.nativeToUnixPathWorkaround).sorted.join("\n")
+      let context = $(i, ret, cmd)
+      var expected = ""
+      case i
+      of 0,5:
+        let htmlFile = htmldocsDir/"mmain.html"
+        check htmlFile in outp # sanity check for `hintSuccessX`
+        assertEquals ret, """
+@@/imp.html
+@@/imp.idx
+dochack.js
+imp.html
+imp.idx
+imp2.html
+imp2.idx
+mmain.html
+mmain.idx
+nimdoc.out.css
+theindex.html""", context
+      of 1: assertEquals ret, """
+dochack.js
+nimdoc.out.css
+tests/nimdoc/imp.html
+tests/nimdoc/imp.idx
+tests/nimdoc/sub/imp.html
+tests/nimdoc/sub/imp.idx
+tests/nimdoc/sub/imp2.html
+tests/nimdoc/sub/imp2.idx
+tests/nimdoc/sub/mmain.html
+tests/nimdoc/sub/mmain.idx
+theindex.html"""
+      of 2, 3: assertEquals ret, """
+dochack.js
+mmain.html
+mmain.idx
+nimdoc.out.css""", context
+      of 4: assertEquals ret, """
+dochack.js
+nimdoc.out.css
+sub/mmain.html
+sub/mmain.idx""", context
+      of 6: assertEquals ret, """
+mmain.html
+nimdoc.out.css""", context
+      else: doAssert false
+
  block: # mstatic_assert
    let (output, exitCode) = runCmd("ccgbugs/mstatic_assert.nim", "-d:caseBad")
    check2 "sizeof(bool) == 2"
@@ -121,7 +198,7 @@ else: # don't run twice the same test
      let cmd = fmt"""{nim} doc -b:{backend} --nimcache:{nimcache} -d:m13129Foo1 "--doccmd:-d:m13129Foo2 --hints:off" --usenimcache --hints:off {file}"""
      check execCmdEx(cmd) == (&"ok1:{backend}\nok2: backend: {backend}\n", 0)
    # checks that --usenimcache works with `nim doc`
-    check fileExists(nimcache / "m13129.html")
+    check fileExists(nimcache / "htmldocs/m13129.html")

    block: # mak sure --backend works with `nim r`
      let cmd = fmt"{nim} r --backend:{mode} --hints:off --nimcache:{nimcache} {file}"
--- a/tests/nimdoc/imp.nim
+++ b/tests/nimdoc/imp.nim
@@ -0,0 +1 @@
+proc fn5*() = discard
--- a/tests/nimdoc/m13129.nim
+++ b/tests/nimdoc/m13129.nim
@@ -17,11 +17,9 @@ proc main*() =
    doAssert defined(m13129Foo2)
    doAssert not defined(nimdoc)
    echo "ok2: backend: " & querySetting(backend)
-    # echo defined(c), defined(js), 

 import std/compilesettings
 when defined nimdoc:
-  # import std/compilesettings
  static:
    doAssert defined(m13129Foo1)
    doAssert not defined(m13129Foo2)
@@ -33,6 +31,6 @@ when isMainModule:
    let cache = querySetting(nimcacheDir)
    doAssert cache.len > 0
    let app = getAppFilename()
-    doAssert app.isRelativeTo(cache)
+    doAssert app.isRelativeTo(cache), $(app, cache)
    doAssert querySetting(projectFull) == currentSourcePath
    echo "ok3"
--- a/tests/nimdoc/sub/imp.nim
+++ b/tests/nimdoc/sub/imp.nim
@@ -0,0 +1 @@
+proc fn4*() = discard
--- a/tests/nimdoc/sub/imp2.nim
+++ b/tests/nimdoc/sub/imp2.nim
@@ -0,0 +1 @@
+proc fn3*() = discard
--- a/tests/nimdoc/sub/mmain.nim
+++ b/tests/nimdoc/sub/mmain.nim
@@ -0,0 +1,8 @@
+{.warning[UnusedImport]: off.}
+
+import ../imp as impa
+import imp as impb
+import imp2
+
+proc fn1*() = discard
+proc fn2*() = discard
--- a/tools/kochdocs.nim
+++ b/tools/kochdocs.nim
@@ -1,6 +1,7 @@
 ## Part of 'koch' responsible for the documentation generation.

 import os, strutils, osproc, sets, pathnorm
+import "../compiler/nimpaths"

 const
  gaCode* = " --doc.googleAnalytics:UA-48159761-1"
@@ -9,7 +10,6 @@ const
  gitUrl = "https://github.com/nim-lang/Nim"
  docHtmlOutput = "doc/html"
  webUploadOutput = "web/upload"
-  docHackDir = "tools/dochack"

 var nimExe*: string

@@ -289,20 +289,18 @@ proc buildPdfDoc*(nimArgs, destPath: string) =
      removeFile(changeFileExt(pdf, "out"))
      removeFile(changeFileExt(d, "tex"))

-proc buildJS() =
-  exec(findNim().quoteShell() & " js -d:release --out:$1 tools/nimblepkglist.nim" %
+proc buildJS(): string =
+  let nim = findNim()
+  exec(nim.quoteShell() & " js -d:release --out:$1 tools/nimblepkglist.nim" %
      [webUploadOutput / "nimblepkglist.js"])
-  exec(findNim().quoteShell() & " js " & (docHackDir / "dochack.nim"))
+  result = getDocHacksJs(nimr = getCurrentDir(), nim)

 proc buildDocs*(args: string) =
-  const
-    docHackJs = "dochack.js"
  let
    a = nimArgs & " " & args
-    docHackJsSource = docHackDir / docHackJs
-    docHackJsDest = docHtmlOutput / docHackJs
+    docHackJsSource = buildJS()
+    docHackJs = docHackJsSource.lastPathPart

-  buildJS()                     # This call generates docHackJsSource
  let docup = webUploadOutput / NimVersion
  createDir(docup)
  buildDocSamples(a, docup)
@@ -313,5 +311,5 @@ proc buildDocs*(args: string) =
  createDir(docHtmlOutput)
  buildDocSamples(nimArgs, docHtmlOutput)
  buildDoc(nimArgs, docHtmlOutput)
-  copyFile(docHackJsSource, docHackJsDest)
+  copyFile(docHackJsSource, docHtmlOutput / docHackJs)
  copyFile(docHackJsSource, docup / docHackJs)