From 97970d9dccb24bb290cdba14275acae62602c832 Mon Sep 17 00:00:00 2001
From: Andrey Makarov
Date: Fri, 14 May 2021 08:30:47 +0300
Subject: [PATCH] `doc2tex`: generate docs to Latex (#17997)
* `doc2tex`: generate docs to Latex
* address some comments
---
compiler/commands.nim | 1 +
compiler/docgen.nim | 22 ++-
compiler/docgen2.nim | 5 +
compiler/main.nim | 25 ++-
compiler/options.nim | 4 +-
compiler/renderverbatim.nim | 21 ---
config/nimdoc.cfg | 1 +
config/nimdoc.tex.cfg | 16 +-
lib/packages/docutils/rst.nim | 3 +-
lib/packages/docutils/rstgen.nim | 60 +++---
.../expected/subdir/subdir_b/utils.html | 2 +-
nimdoc/testproject/expected/testproject.html | 172 +++++++++---------
12 files changed, 171 insertions(+), 161 deletions(-)
diff --git a/compiler/commands.nim b/compiler/commands.nim
index 16b85275c2..b9c8e9eb37 100644
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -444,6 +444,7 @@ proc parseCommand*(command: string): Command =
of "e": cmdNimscript
of "doc0": cmdDoc0
of "doc2", "doc": cmdDoc2
+ of "doc2tex": cmdDoc2tex
of "rst2html": cmdRst2html
of "rst2tex": cmdRst2tex
of "jsondoc0": cmdJsondoc0
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index fac4a8defe..23e6ba10da 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -169,14 +169,17 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
else:
result = getOutFile(conf, filename, ext)
-proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil): PDoc =
+proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
+
+proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
+ outExt: string = HtmlExt, module: PSym = nil): PDoc =
declareClosures()
new(result)
result.module = module
result.conf = conf
result.cache = cache
result.outDir = conf.outDir.string
- initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
+ initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
conf.configVars, filename.string,
{roSupportRawDirective, roSupportMarkdown,
roPreferMarkdown, roNimFile},
@@ -249,7 +252,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
template dispA(conf: ConfigRef; dest: var string, xml, tex: string,
args: openArray[string]) =
- if conf.cmd != cmdRst2tex: dest.addf(xml, args)
+ if not conf.isLatexCmd: dest.addf(xml, args)
else: dest.addf(tex, args)
proc getVarIdx(varnames: openArray[string], id: string): int =
@@ -551,7 +554,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
let id = $d.listingCounter
dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
var dest2 = ""
- renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+ renderNimCode(dest2, code, d.target)
dest.add dest2
dest.add(d.config.getOrDefault"doc.listing_end" % id)
return rsRunnable
@@ -821,7 +824,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
inc(d.id)
let
- plainNameEsc = xmltree.escape(plainName.strip)
+ plainNameEsc = esc(d.target, plainName.strip)
+ uniqueName = if k in routineKinds: plainNameEsc else: name
cleanPlainSymbol = renderPlainSymbolName(nameNode)
complexSymbol = complexName(k, n, cleanPlainSymbol)
plainSymbolEnc = encodeUrl(cleanPlainSymbol)
@@ -835,7 +839,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
d.section[k].add(getConfigVar(d.conf, "doc.item") %
- ["name", name, "header", result, "desc", comm, "itemID", $d.id,
+ ["name", name, "uniqueName", uniqueName,
+ "header", result, "desc", comm, "itemID", $d.id,
"header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
"itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc,
"itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
@@ -1199,7 +1204,8 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
var shouldSort = i in routineKinds and groupedToc
genSection(d, i, shouldSort)
toc.add(d.toc[i])
- if toc != "":
+ if toc != "" or d.target == outLatex:
+ # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set
toc = getConfigVar(d.conf, "doc.toc") % ["content", toc]
for i in TSymKind: code.add(d.section[i])
@@ -1217,7 +1223,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
"\\\\\\vspace{0.5em}\\large $1", [d.meta[metaSubtitle]])
var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
- let bodyname = if d.hasToc and not d.isPureRst:
+ let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd:
groupsection.setLen 0
"doc.body_toc_group"
elif d.hasToc: "doc.body_toc"
diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim
index cfbb33156f..d077ddc672 100644
--- a/compiler/docgen2.nim
+++ b/compiler/docgen2.nim
@@ -71,10 +71,15 @@ template myOpenImpl(ext: untyped) {.dirty.} =
proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
myOpenImpl(HtmlExt)
+proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+ myOpenImpl(TexExt)
+
proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
myOpenImpl(JsonExt)
const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
+const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
+ close = close)
const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
close = closeJson)
diff --git a/compiler/main.nim b/compiler/main.nim
index db5dd439ee..e2b5f4b676 100644
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -69,12 +69,15 @@ proc commandCheck(graph: ModuleGraph) =
writeRodFiles(graph)
when not defined(leanCompiler):
- proc commandDoc2(graph: ModuleGraph; json: bool) =
+ proc commandDoc2(graph: ModuleGraph; ext: string) =
handleDocOutputOptions graph.config
graph.config.setErrorMaxHighMaybe
semanticPasses(graph)
- if json: registerPass(graph, docgen2JsonPass)
- else: registerPass(graph, docgen2Pass)
+ case ext:
+ of TexExt: registerPass(graph, docgen2TexPass)
+ of JsonExt: registerPass(graph, docgen2JsonPass)
+ of HtmlExt: registerPass(graph, docgen2Pass)
+ else: doAssert false, $ext
compileProject(graph)
finishDoc2Pass(graph.config.projectName)
@@ -249,7 +252,8 @@ proc mainCommand*(graph: ModuleGraph) =
conf.quitOrRaise "compiler wasn't built with documentation generator"
else:
wantMainModule(conf)
- loadConfigs(DocConfig, cache, conf, graph.idgen)
+ let docConf = if conf.cmd == cmdDoc2tex: DocTexConfig else: DocConfig
+ loadConfigs(docConf, cache, conf, graph.idgen)
defineSymbol(conf.symbols, "nimdoc")
body
@@ -285,7 +289,7 @@ proc mainCommand*(graph: ModuleGraph) =
# of labels links in doc comments, e.g. for random.rand:
# ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
# ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
- commandDoc2(graph, false)
+ commandDoc2(graph, HtmlExt)
if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
commandBuildIndex(conf, $conf.outDir)
of cmdRst2html:
@@ -299,7 +303,7 @@ proc mainCommand*(graph: ModuleGraph) =
else:
loadConfigs(DocConfig, cache, conf, graph.idgen)
commandRst2Html(cache, conf)
- of cmdRst2tex:
+ of cmdRst2tex, cmdDoc2tex:
for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX,
warnLanguageXNotSupported,
warnFieldXNotSupported, warnRstStyle]:
@@ -307,10 +311,13 @@ proc mainCommand*(graph: ModuleGraph) =
when defined(leanCompiler):
conf.quitOrRaise "compiler wasn't built with documentation generator"
else:
- loadConfigs(DocTexConfig, cache, conf, graph.idgen)
- commandRst2TeX(cache, conf)
+ if conf.cmd == cmdRst2tex:
+ loadConfigs(DocTexConfig, cache, conf, graph.idgen)
+ commandRst2TeX(cache, conf)
+ else:
+ docLikeCmd commandDoc2(graph, TexExt)
of cmdJsondoc0: docLikeCmd commandJson(cache, conf)
- of cmdJsondoc: docLikeCmd commandDoc2(graph, true)
+ of cmdJsondoc: docLikeCmd commandDoc2(graph, JsonExt)
of cmdCtags: docLikeCmd commandTags(cache, conf)
of cmdBuildindex: docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
of cmdGendepend: commandGenDepend(graph)
diff --git a/compiler/options.nim b/compiler/options.nim
index 09eae49e6a..9cbb747c97 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -144,6 +144,7 @@ type
cmdNimscript # evaluate nimscript
cmdDoc0
cmdDoc2
+ cmdDoc2tex
cmdRst2html # convert a reStructuredText file to HTML
cmdRst2tex # convert a reStructuredText file to TeX
cmdJsondoc0
@@ -160,7 +161,8 @@ type
const
cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun}
- cmdDocLike* = {cmdDoc0, cmdDoc2, cmdJsondoc0, cmdJsondoc, cmdCtags, cmdBuildindex}
+ cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
+ cmdCtags, cmdBuildindex}
type
TStringSeq* = seq[string]
diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim
index 02d4058442..a20c8873d9 100644
--- a/compiler/renderverbatim.nim
+++ b/compiler/renderverbatim.nim
@@ -1,8 +1,6 @@
import strutils
-from xmltree import addEscaped
import ast, options, msgs
-import packages/docutils/highlite
const isDebug = false
when isDebug:
@@ -131,22 +129,3 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
lastNonemptyPos = result.len
result.setLen lastNonemptyPos
-proc renderNimCode*(result: var string, code: string, isLatex = false) =
- var toknizr: GeneralTokenizer
- initGeneralTokenizer(toknizr, code)
- var buf = ""
- template append(kind, val) =
- buf.setLen 0
- buf.addEscaped(val)
- let class = tokenClassToStr[kind]
- if isLatex:
- result.addf "\\span$1{$2}", [class, buf]
- else:
- result.addf "$2", [class, buf]
- while true:
- getNextToken(toknizr, langNim)
- case toknizr.kind
- of gtEof: break # End Of File (or string)
- else:
- # TODO: avoid alloc; maybe toOpenArray
- append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))
diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg
index 82bd9cc219..ed1b346a22 100644
--- a/config/nimdoc.cfg
+++ b/config/nimdoc.cfg
@@ -42,6 +42,7 @@ doc.section.toc2 = """
# * $itemSymOrID: the symbolic name or the ID if that is not unique.
# * $itemSymOrIDEnc: quoted version for URLs or attributes.
# * $name: reduced name of the item.
+# * $uniqueName: name with parameters for routine types or $name for others.
# * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used).
doc.item = """
diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg
index b1c02c5a0d..b19b256787 100644
--- a/config/nimdoc.tex.cfg
+++ b/config/nimdoc.tex.cfg
@@ -8,22 +8,24 @@ split.item.toc = "20"
# after this number of characters
doc.section = """
-\chapter{$sectionTitle}\label{$sectionID}
-\begin{description}
+\rsthA{$sectionTitle}\label{$sectionID}
$content
-\end{description}
"""
doc.section.toc = ""
# $sectionID $sectionTitleID $sectionTitle $content
doc.item = """
-\item[\texttt{$header}\label{$itemID}]\mbox{~}\\*
+
+\phantomsection\addcontentsline{toc}{subsection}{$uniqueName}
+
+\begin{rstpre}
+$header
+\end{rstpre}
$desc
"""
doc.item.toc = ""
-# \item $name\ref{$itemID}
doc.toc = r"\tableofcontents \newpage"
@@ -38,6 +40,10 @@ $moduledesc
$content
"""
+# $1 - number of listing in document, $2 - language (e.g. langNim), $3 - anchor
+doc.listing_start = "\\begin{rstpre}\n"
+doc.listing_end = "\n\\end{rstpre}\n"
+
doc.file = """
% This file was generated by Nim.
% Generated: $date $time UTC
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index ea0d62e045..181929c091 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -26,7 +26,8 @@
## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and
## `nim rst2html`:cmd: for ``*.rst`` files
##
-## .. [#latex] command `nim rst2tex`:cmd: for ``*.rst``.
+## .. [#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and
+## `nim rst2tex`:cmd: for ``*.rst``.
##
## If you are new to RST please consider reading the following:
##
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index 40ed889545..5ab6f28ee9 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -179,7 +179,9 @@ proc writeIndexFile*(g: var RstGenerator, outfile: string) =
## If the index is empty the file won't be created.
if g.theIndex.len > 0: writeFile(outfile, g.theIndex)
-proc addXmlChar(dest: var string, c: char) =
+proc addHtmlChar(dest: var string, c: char) =
+ # Escapes HTML characters. Note that single quote ' is not escaped as
+ # ' -- unlike XML (for standards pre HTML5 it was even forbidden).
case c
of '&': add(dest, "&")
of '<': add(dest, "<")
@@ -195,26 +197,19 @@ proc addRtfChar(dest: var string, c: char) =
else: add(dest, c)
proc addTexChar(dest: var string, c: char) =
+ # Escapes 10 special Latex characters. Note that [, ], and ` are not
+ # considered as such. TODO: neither is @, am I wrong?
case c
- of '_': add(dest, "\\_")
- of '{': add(dest, "\\symbol{123}")
- of '}': add(dest, "\\symbol{125}")
- of '[': add(dest, "\\symbol{91}")
- of ']': add(dest, "\\symbol{93}")
- of '\\': add(dest, "\\symbol{92}")
- of '$': add(dest, "\\$")
- of '&': add(dest, "\\&")
- of '#': add(dest, "\\#")
- of '%': add(dest, "\\%")
- of '~': add(dest, "\\symbol{126}")
- of '@': add(dest, "\\symbol{64}")
- of '^': add(dest, "\\symbol{94}")
- of '`': add(dest, "\\symbol{96}")
+ of '_', '{', '}', '$', '&', '#', '%': add(dest, "\\" & c)
+ # \~ and \^ have a special meaning unless they are followed by {}
+ of '~', '^': add(dest, "\\" & c & "{}")
+ # add {} to avoid gobbling up space by \textbackslash
+ of '\\': add(dest, "\\textbackslash{}")
else: add(dest, c)
proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} =
case target
- of outHtml: addXmlChar(dest, c)
+ of outHtml: addHtmlChar(dest, c)
of outLatex: addTexChar(dest, c)
proc addSplitter(target: OutputTarget; dest: var string) {.inline.} =
@@ -987,6 +982,25 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string,
"" & (
d.config.getOrDefault"doc.listing_button" % id)
+proc renderCodeLang*(result: var string, lang: SourceLanguage, code: string,
+ target: OutputTarget) =
+ var g: GeneralTokenizer
+ initGeneralTokenizer(g, code)
+ while true:
+ getNextToken(g, lang)
+ case g.kind
+ of gtEof: break
+ of gtNone, gtWhitespace:
+ add(result, substr(code, g.start, g.length + g.start - 1))
+ else:
+ dispA(target, result, "$1", "\\span$2{$1}", [
+ esc(target, substr(code, g.start, g.length+g.start-1)),
+ tokenClassToStr[g.kind]])
+ deinitGeneralTokenizer(g)
+
+proc renderNimCode*(result: var string, code: string, target: OutputTarget) =
+ renderCodeLang(result, langNim, code, target)
+
proc renderCode(d: PDoc, n: PRstNode, result: var string) =
## Renders a code (code block or inline code), appending it to `result`.
##
@@ -1028,19 +1042,7 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) =
d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
for letter in m.text: escChar(d.target, result, letter)
else:
- var g: GeneralTokenizer
- initGeneralTokenizer(g, m.text)
- while true:
- getNextToken(g, params.lang)
- case g.kind
- of gtEof: break
- of gtNone, gtWhitespace:
- add(result, substr(m.text, g.start, g.length + g.start - 1))
- else:
- dispA(d.target, result, "$1", "\\span$2{$1}", [
- esc(d.target, substr(m.text, g.start, g.length+g.start-1)),
- tokenClassToStr[g.kind]])
- deinitGeneralTokenizer(g)
+ renderCodeLang(result, params.lang, m.text, d.target)
dispA(d.target, result, blockEnd, blockEnd)
proc renderContainer(d: PDoc, n: PRstNode, result: var string) =
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index c1ee9c49d5..794066cae3 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -208,7 +208,7 @@ constructor.
this should be shown in utils.html
Example:
-assert 3*2 == 6
ditto
+assert 3*2 == 6
ditto
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 70c5b695a6..f23f6bb42e 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -133,13 +133,13 @@ window.addEventListener('DOMContentLoaded', main);
Consts
- C_A
+ title="C_A = 0x7FF0000000000000'f64">C_A
- C_B
+ title="C_B = 0o377'i8">C_B
- C_C
+ title="C_C = 0o277'i8">C_C
- C_D
+ title="C_D = 0o177777'i16">C_D
@@ -420,21 +420,21 @@ window.addEventListener('DOMContentLoaded', main);
This is the top level module.
Example:
-import testproject
-import subdir / subdir_b / utils
-doAssert bar(3, 4) == 7
-foo(enumValueA, enumValueB)
-
-for x in "xx": discard
top2
+import testproject
+import subdir / subdir_b / utils
+doAssert bar(3, 4) == 7
+foo(enumValueA, enumValueB)
+
+for x in "xx": discard
top2
Example:
-import testproject
-discard "in top2"
top2 after
+import testproject
+discard "in top2"
top2 after
Example:
-import testproject
-discard "in top3"
top3 after
+import testproject
+discard "in top3"
top3 after
Example:
-import testproject
-assert 3*2 == 6
+import testproject
+assert 3*2 == 6
@@ -574,7 +574,7 @@ My someFunc. Stuff in fromUtilsGen is called
Example:
-discard 1
+discard 1
@@ -597,7 +597,7 @@ cz1
cz2
Example:
-discard "in cz2"
+discard "in cz2"
@@ -648,7 +648,7 @@ cz8
Example:
-doAssert 1 + 1 == 2
+doAssert 1 + 1 == 2
@@ -657,7 +657,7 @@ cz8
Example: cmd: -d:foobar
-discard 1
cz10
+discard 1
cz10
@@ -666,7 +666,7 @@ cz8
Example:
-discard 1
+discard 1
@@ -675,7 +675,7 @@ cz8
Example:
-discard 1
+discard 1
@@ -700,7 +700,7 @@ cz13
cz17 rest
Example:
-discard 1
rest
+discard 1
rest
@@ -709,21 +709,21 @@ cz17 rest
cp1
Example:
-doAssert 1 == 1
c4
+doAssert 1 == 1
c4
Example:
-
-
-
+]##
+discard "c9"
+
@@ -732,9 +732,9 @@ this is a nested doc comment
Some proc
Example:
-discard "foo() = " & $[1]
-
should be still in
@@ -1012,13 +1012,13 @@ cz15
Example:
discard
Example:
-discard 3
+discard 3
Example:
-discard 4
ok5 ok5b
+discard 4
ok5 ok5b
Example:
-assert true
+assert true
Example:
-discard 1
in or out?
+discard 1
in or out?
@@ -1027,7 +1027,7 @@ cz15
Example:
-discard 2
+discard 2