diff --git a/compiler/commands.nim b/compiler/commands.nim index 16b85275c2..b9c8e9eb37 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -444,6 +444,7 @@ proc parseCommand*(command: string): Command = of "e": cmdNimscript of "doc0": cmdDoc0 of "doc2", "doc": cmdDoc2 + of "doc2tex": cmdDoc2tex of "rst2html": cmdRst2html of "rst2tex": cmdRst2tex of "jsondoc0": cmdJsondoc0 diff --git a/compiler/docgen.nim b/compiler/docgen.nim index fac4a8defe..23e6ba10da 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -169,14 +169,17 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile, else: result = getOutFile(conf, filename, ext) -proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil): PDoc = +proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex} + +proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, + outExt: string = HtmlExt, module: PSym = nil): PDoc = declareClosures() new(result) result.module = module result.conf = conf result.cache = cache result.outDir = conf.outDir.string - initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex), + initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml), conf.configVars, filename.string, {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roNimFile}, @@ -249,7 +252,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, template dispA(conf: ConfigRef; dest: var string, xml, tex: string, args: openArray[string]) = - if conf.cmd != cmdRst2tex: dest.addf(xml, args) + if not conf.isLatexCmd: dest.addf(xml, args) else: dest.addf(tex, args) proc getVarIdx(varnames: openArray[string], id: string): int = @@ -551,7 +554,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string, let id = $d.listingCounter dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""]) var dest2 = "" - renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex) + renderNimCode(dest2, code, d.target) dest.add dest2 dest.add(d.config.getOrDefault"doc.listing_end" % id) return rsRunnable @@ -821,7 +824,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = inc(d.id) let - plainNameEsc = xmltree.escape(plainName.strip) + plainNameEsc = esc(d.target, plainName.strip) + uniqueName = if k in routineKinds: plainNameEsc else: name cleanPlainSymbol = renderPlainSymbolName(nameNode) complexSymbol = complexName(k, n, cleanPlainSymbol) plainSymbolEnc = encodeUrl(cleanPlainSymbol) @@ -835,7 +839,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) d.section[k].add(getConfigVar(d.conf, "doc.item") % - ["name", name, "header", result, "desc", comm, "itemID", $d.id, + ["name", name, "uniqueName", uniqueName, + "header", result, "desc", comm, "itemID", $d.id, "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol, "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc, "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc, @@ -1199,7 +1204,8 @@ proc genOutFile(d: PDoc, groupedToc = false): string = var shouldSort = i in routineKinds and groupedToc genSection(d, i, shouldSort) toc.add(d.toc[i]) - if toc != "": + if toc != "" or d.target == outLatex: + # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set toc = getConfigVar(d.conf, "doc.toc") % ["content", toc] for i in TSymKind: code.add(d.section[i]) @@ -1217,7 +1223,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = "\\\\\\vspace{0.5em}\\large $1", [d.meta[metaSubtitle]]) var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection") - let bodyname = if d.hasToc and not d.isPureRst: + let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd: groupsection.setLen 0 "doc.body_toc_group" elif d.hasToc: "doc.body_toc" diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index cfbb33156f..d077ddc672 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -71,10 +71,15 @@ template myOpenImpl(ext: untyped) {.dirty.} = proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = myOpenImpl(HtmlExt) +proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = + myOpenImpl(TexExt) + proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = myOpenImpl(JsonExt) const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close) +const docgen2TexPass* = makePass(open = myOpenTex, process = processNode, + close = close) const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson, close = closeJson) diff --git a/compiler/main.nim b/compiler/main.nim index db5dd439ee..e2b5f4b676 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -69,12 +69,15 @@ proc commandCheck(graph: ModuleGraph) = writeRodFiles(graph) when not defined(leanCompiler): - proc commandDoc2(graph: ModuleGraph; json: bool) = + proc commandDoc2(graph: ModuleGraph; ext: string) = handleDocOutputOptions graph.config graph.config.setErrorMaxHighMaybe semanticPasses(graph) - if json: registerPass(graph, docgen2JsonPass) - else: registerPass(graph, docgen2Pass) + case ext: + of TexExt: registerPass(graph, docgen2TexPass) + of JsonExt: registerPass(graph, docgen2JsonPass) + of HtmlExt: registerPass(graph, docgen2Pass) + else: doAssert false, $ext compileProject(graph) finishDoc2Pass(graph.config.projectName) @@ -249,7 +252,8 @@ proc mainCommand*(graph: ModuleGraph) = conf.quitOrRaise "compiler wasn't built with documentation generator" else: wantMainModule(conf) - loadConfigs(DocConfig, cache, conf, graph.idgen) + let docConf = if conf.cmd == cmdDoc2tex: DocTexConfig else: DocConfig + loadConfigs(docConf, cache, conf, graph.idgen) defineSymbol(conf.symbols, "nimdoc") body @@ -285,7 +289,7 @@ proc mainCommand*(graph: ModuleGraph) = # of labels links in doc comments, e.g. for random.rand: # ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer # ## * `rand proc<#rand,Rand,range[]>`_ that returns a float - commandDoc2(graph, false) + commandDoc2(graph, HtmlExt) if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions: commandBuildIndex(conf, $conf.outDir) of cmdRst2html: @@ -299,7 +303,7 @@ proc mainCommand*(graph: ModuleGraph) = else: loadConfigs(DocConfig, cache, conf, graph.idgen) commandRst2Html(cache, conf) - of cmdRst2tex: + of cmdRst2tex, cmdDoc2tex: for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX, warnLanguageXNotSupported, warnFieldXNotSupported, warnRstStyle]: @@ -307,10 +311,13 @@ proc mainCommand*(graph: ModuleGraph) = when defined(leanCompiler): conf.quitOrRaise "compiler wasn't built with documentation generator" else: - loadConfigs(DocTexConfig, cache, conf, graph.idgen) - commandRst2TeX(cache, conf) + if conf.cmd == cmdRst2tex: + loadConfigs(DocTexConfig, cache, conf, graph.idgen) + commandRst2TeX(cache, conf) + else: + docLikeCmd commandDoc2(graph, TexExt) of cmdJsondoc0: docLikeCmd commandJson(cache, conf) - of cmdJsondoc: docLikeCmd commandDoc2(graph, true) + of cmdJsondoc: docLikeCmd commandDoc2(graph, JsonExt) of cmdCtags: docLikeCmd commandTags(cache, conf) of cmdBuildindex: docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile) of cmdGendepend: commandGenDepend(graph) diff --git a/compiler/options.nim b/compiler/options.nim index 09eae49e6a..9cbb747c97 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -144,6 +144,7 @@ type cmdNimscript # evaluate nimscript cmdDoc0 cmdDoc2 + cmdDoc2tex cmdRst2html # convert a reStructuredText file to HTML cmdRst2tex # convert a reStructuredText file to TeX cmdJsondoc0 @@ -160,7 +161,8 @@ type const cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun} - cmdDocLike* = {cmdDoc0, cmdDoc2, cmdJsondoc0, cmdJsondoc, cmdCtags, cmdBuildindex} + cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc, + cmdCtags, cmdBuildindex} type TStringSeq* = seq[string] diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim index 02d4058442..a20c8873d9 100644 --- a/compiler/renderverbatim.nim +++ b/compiler/renderverbatim.nim @@ -1,8 +1,6 @@ import strutils -from xmltree import addEscaped import ast, options, msgs -import packages/docutils/highlite const isDebug = false when isDebug: @@ -131,22 +129,3 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string = lastNonemptyPos = result.len result.setLen lastNonemptyPos -proc renderNimCode*(result: var string, code: string, isLatex = false) = - var toknizr: GeneralTokenizer - initGeneralTokenizer(toknizr, code) - var buf = "" - template append(kind, val) = - buf.setLen 0 - buf.addEscaped(val) - let class = tokenClassToStr[kind] - if isLatex: - result.addf "\\span$1{$2}", [class, buf] - else: - result.addf "$2", [class, buf] - while true: - getNextToken(toknizr, langNim) - case toknizr.kind - of gtEof: break # End Of File (or string) - else: - # TODO: avoid alloc; maybe toOpenArray - append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1)) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 82bd9cc219..ed1b346a22 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -42,6 +42,7 @@ doc.section.toc2 = """ # * $itemSymOrID: the symbolic name or the ID if that is not unique. # * $itemSymOrIDEnc: quoted version for URLs or attributes. # * $name: reduced name of the item. +# * $uniqueName: name with parameters for routine types or $name for others. # * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used). doc.item = """ diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index b1c02c5a0d..b19b256787 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -8,22 +8,24 @@ split.item.toc = "20" # after this number of characters doc.section = """ -\chapter{$sectionTitle}\label{$sectionID} -\begin{description} +\rsthA{$sectionTitle}\label{$sectionID} $content -\end{description} """ doc.section.toc = "" # $sectionID $sectionTitleID $sectionTitle $content doc.item = """ -\item[\texttt{$header}\label{$itemID}]\mbox{~}\\* + +\phantomsection\addcontentsline{toc}{subsection}{$uniqueName} + +\begin{rstpre} +$header +\end{rstpre} $desc """ doc.item.toc = "" -# \item $name\ref{$itemID} doc.toc = r"\tableofcontents \newpage" @@ -38,6 +40,10 @@ $moduledesc $content """ +# $1 - number of listing in document, $2 - language (e.g. langNim), $3 - anchor +doc.listing_start = "\\begin{rstpre}\n" +doc.listing_end = "\n\\end{rstpre}\n" + doc.file = """ % This file was generated by Nim. % Generated: $date $time UTC diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index ea0d62e045..181929c091 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -26,7 +26,8 @@ ## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and ## `nim rst2html`:cmd: for ``*.rst`` files ## -## .. [#latex] command `nim rst2tex`:cmd: for ``*.rst``. +## .. [#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and +## `nim rst2tex`:cmd: for ``*.rst``. ## ## If you are new to RST please consider reading the following: ## diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 40ed889545..5ab6f28ee9 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -179,7 +179,9 @@ proc writeIndexFile*(g: var RstGenerator, outfile: string) = ## If the index is empty the file won't be created. if g.theIndex.len > 0: writeFile(outfile, g.theIndex) -proc addXmlChar(dest: var string, c: char) = +proc addHtmlChar(dest: var string, c: char) = + # Escapes HTML characters. Note that single quote ' is not escaped as + # ' -- unlike XML (for standards pre HTML5 it was even forbidden). case c of '&': add(dest, "&") of '<': add(dest, "<") @@ -195,26 +197,19 @@ proc addRtfChar(dest: var string, c: char) = else: add(dest, c) proc addTexChar(dest: var string, c: char) = + # Escapes 10 special Latex characters. Note that [, ], and ` are not + # considered as such. TODO: neither is @, am I wrong? case c - of '_': add(dest, "\\_") - of '{': add(dest, "\\symbol{123}") - of '}': add(dest, "\\symbol{125}") - of '[': add(dest, "\\symbol{91}") - of ']': add(dest, "\\symbol{93}") - of '\\': add(dest, "\\symbol{92}") - of '$': add(dest, "\\$") - of '&': add(dest, "\\&") - of '#': add(dest, "\\#") - of '%': add(dest, "\\%") - of '~': add(dest, "\\symbol{126}") - of '@': add(dest, "\\symbol{64}") - of '^': add(dest, "\\symbol{94}") - of '`': add(dest, "\\symbol{96}") + of '_', '{', '}', '$', '&', '#', '%': add(dest, "\\" & c) + # \~ and \^ have a special meaning unless they are followed by {} + of '~', '^': add(dest, "\\" & c & "{}") + # add {} to avoid gobbling up space by \textbackslash + of '\\': add(dest, "\\textbackslash{}") else: add(dest, c) proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} = case target - of outHtml: addXmlChar(dest, c) + of outHtml: addHtmlChar(dest, c) of outLatex: addTexChar(dest, c) proc addSplitter(target: OutputTarget; dest: var string) {.inline.} = @@ -987,6 +982,25 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string, "" & ( d.config.getOrDefault"doc.listing_button" % id) +proc renderCodeLang*(result: var string, lang: SourceLanguage, code: string, + target: OutputTarget) = + var g: GeneralTokenizer + initGeneralTokenizer(g, code) + while true: + getNextToken(g, lang) + case g.kind + of gtEof: break + of gtNone, gtWhitespace: + add(result, substr(code, g.start, g.length + g.start - 1)) + else: + dispA(target, result, "$1", "\\span$2{$1}", [ + esc(target, substr(code, g.start, g.length+g.start-1)), + tokenClassToStr[g.kind]]) + deinitGeneralTokenizer(g) + +proc renderNimCode*(result: var string, code: string, target: OutputTarget) = + renderCodeLang(result, langNim, code, target) + proc renderCode(d: PDoc, n: PRstNode, result: var string) = ## Renders a code (code block or inline code), appending it to `result`. ## @@ -1028,19 +1042,7 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) = d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr) for letter in m.text: escChar(d.target, result, letter) else: - var g: GeneralTokenizer - initGeneralTokenizer(g, m.text) - while true: - getNextToken(g, params.lang) - case g.kind - of gtEof: break - of gtNone, gtWhitespace: - add(result, substr(m.text, g.start, g.length + g.start - 1)) - else: - dispA(d.target, result, "$1", "\\span$2{$1}", [ - esc(d.target, substr(m.text, g.start, g.length+g.start-1)), - tokenClassToStr[g.kind]]) - deinitGeneralTokenizer(g) + renderCodeLang(result, params.lang, m.text, d.target) dispA(d.target, result, blockEnd, blockEnd) proc renderContainer(d: PDoc, n: PRstNode, result: var string) = diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index c1ee9c49d5..794066cae3 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -208,7 +208,7 @@ constructor. this should be shown in utils.html
Example:
-assert 3*2 == 6ditto +
assert 3*2 == 6ditto diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 70c5b695a6..f23f6bb42e 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -133,13 +133,13 @@ window.addEventListener('DOMContentLoaded', main); Consts
This is the top level module.
Example:
-import testproject -import subdir / subdir_b / utils -doAssert bar(3, 4) == 7 -foo(enumValueA, enumValueB) -# bug #11078 -for x in "xx": discardtop2 +
import testproject +import subdir / subdir_b / utils +doAssert bar(3, 4) == 7 +foo(enumValueA, enumValueB) +# bug #11078 +for x in "xx": discardtop2
Example:
-import testproject -discard "in top2"top2 after +
import testproject +discard "in top2"top2 after
Example:
-import testproject -discard "in top3"top3 after +
import testproject +discard "in top3"top3 after
Example:
-import testproject -assert 3*2 == 6+
import testproject +assert 3*2 == 6
Example:
-discard 1+
discard 1@@ -597,7 +597,7 @@ cz1 cz2
Example:
-discard "in cz2"+
discard "in cz2"@@ -648,7 +648,7 @@ cz8
Example:
-doAssert 1 + 1 == 2+
doAssert 1 + 1 == 2@@ -657,7 +657,7 @@ cz8
Example: cmd: -d:foobar
-discard 1cz10 +
discard 1cz10 @@ -666,7 +666,7 @@ cz8
Example:
-discard 1+
discard 1@@ -675,7 +675,7 @@ cz8
Example:
-discard 1+
discard 1@@ -700,7 +700,7 @@ cz13 cz17 rest
Example:
-discard 1rest +
discard 1rest @@ -709,21 +709,21 @@ cz17 rest cp1
Example:
-doAssert 1 == 1 # regular comments work herec4 +
doAssert 1 == 1 # regular comments work herec4
Example:
-# c5 regular comments before 1st token work -# regular comment -#[ ++]## +discard "c9" +# also work after @@ -732,9 +732,9 @@ this is a nested doc comment Some proc# c5 regular comments before 1st token work +# regular comment +#[ nested regular comment -]# -doAssert 2 == 2 # c8 -## this is a non-nested doc comment +]# +doAssert 2 == 2 # c8 +## this is a non-nested doc comment -##[ +##[ this is a nested doc comment -]## -discard "c9" -# also work after
Example:
-discard "foo() = " & $[1] -#[ -0: let's also add some broken html to make sure this won't break in future +should be still in @@ -1012,13 +1012,13 @@ cz15discard "foo() = " & $[1] +#[ +0: let's also add some broken html to make sure this won't break in future 1: </span> 2: </span> 3: </span @@ -782,7 +782,7 @@ the c printf. etc.@@ -896,9 +896,9 @@ foo3low2(2) # => -9223372036854775808Example:
-discard "in low2"+discard "in low2"@@ -791,41 +791,41 @@ the c printf. etc.Example:
-## mullitline string litterals are tricky as their indentation can span -## below that of the runnableExamples -let s1a = """ ++let s5 = """ x +""" +let s6 = """ "" +""" +let s7 = """""""""" +let s8 = ["""""""""", """ + """ ] +discard +# should be in## mullitline string litterals are tricky as their indentation can span +## below that of the runnableExamples +let s1a = """ should appear at indent 0 at indent 2 at indent 0 -""" -# make sure this works too -let s1b = """start at same line +""" +# make sure this works too +let s1b = """start at same line at indent 2 at indent 0 -""" # comment after -let s2 = """sandwich """ -let s3 = """""" -when false: - let s5 = """ - in s5 """ +""" # comment after +let s2 = """sandwich """ +let s3 = """""" +when false: + let s5 = """ + in s5 """ -let s3b = [""" +let s3b = [""" %!? #[...] # inside a multiline ... -""", "foo"] +""", "foo"] -## make sure handles trailing spaces -let s4 = """ -""" +## make sure handles trailing spaces +let s4 = """ +""" -let s5 = """ x -""" -let s6 = """ "" -""" -let s7 = """""""""" -let s8 = ["""""""""", """ - """ ] -discard -# should be inExample:
-# ok1 -assert 1 == 1 -# ok2+# ok1 +assert 1 == 1 +# ok2@@ -914,7 +914,7 @@ foo1 foo2Example:
-discard # bar+discard # bar@@ -935,9 +935,9 @@ foo2Example:
-discard 1cz16 after +discard 1cz16 afterExample:
-doAssert 2 == 1 + 1+doAssert 2 == 1 + 1@@ -979,20 +979,20 @@ This does nothingExample:
-import std/strutils -## issue #8871 preserve formatting -## line doc comment -# bar -doAssert "'foo" == "'foo" -##[ +should be still in +doAssert: not "foo".startsWith "ba" +block: + discard 0xff # elu par cette crapule +# should be inimport std/strutils +## issue #8871 preserve formatting +## line doc comment +# bar +doAssert "'foo" == "'foo" +##[ foo bar -]## +]## -doAssert: not "foo".startsWith "ba" -block: - discard 0xff # elu par cette crapule -# should be in
Example:
discard
Example:
-discard 3+
discard 3
Example:
-discard 4ok5 ok5b +
discard 4ok5 ok5b
Example:
-assert true+
assert true
Example:
-discard 1in or out? +
discard 1in or out? @@ -1027,7 +1027,7 @@ cz15
Example:
-discard 2+
discard 2