diff --git a/compiler/ast.nim b/compiler/ast.nim index 97aa3a9927..d13487e9cf 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1030,6 +1030,7 @@ const nkFloatLiterals* = {nkFloatLit..nkFloat128Lit} nkLambdaKinds* = {nkLambda, nkDo} declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef} + routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef} procDefs* = nkLambdaKinds + declarativeDefs nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice} diff --git a/compiler/docgen.nim b/compiler/docgen.nim index d1e2d42c55..f2dec41b3a 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -17,7 +17,7 @@ import packages/docutils/rst, packages/docutils/rstgen, json, xmltree, cgi, trees, types, typesrenderer, astalgo, lineinfos, intsets, - pathutils, trees, tables, nimpaths + pathutils, trees, tables, nimpaths, renderverbatim const exportSection = skField @@ -315,7 +315,11 @@ proc genComment(d: PDoc, n: PNode): string = result = "" var dummyHasToc: bool if n.comment.len > 0: - renderRstToOut(d[], parseRst(n.comment, toFullPath(d.conf, n.info), + var comment2 = n.comment + when false: + # RFC: to preseve newlines in comments, this would work: + comment2 = comment2.replace("\n", "\n\n") + renderRstToOut(d[], parseRst(comment2, toFullPath(d.conf, n.info), toLinenumber(n.info), toColumn(n.info), dummyHasToc, d.options, d.conf), result) @@ -494,8 +498,8 @@ proc runAllExamples(d: PDoc) = rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string]) # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove -proc prepareExample(d: PDoc; n: PNode): string = - ## returns `rdoccmd` for this runnableExamples +proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] = + ## returns `rdoccmd` and source code for this runnableExamples var rdoccmd = "" if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid") if n.len == 3: @@ -512,10 +516,11 @@ proc prepareExample(d: PDoc; n: PNode): string = docComment, newTree(nkImportStmt, newStrNode(nkStrLit, d.filename))) runnableExamples.info = n.info - + let ret = extractRunnableExamplesSource(d.conf, n) for a in n.lastSon: runnableExamples.add a + # we could also use `ret` instead here, to keep sources verbatim writeExample(d, runnableExamples, rdoccmd) - result = rdoccmd + result = (rdoccmd, ret) when false: proc extractImports(n: PNode; result: PNode) = if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}: @@ -529,39 +534,52 @@ proc prepareExample(d: PDoc; n: PNode): string = for imp in imports: runnableExamples.add imp runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon) -proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previousIsRunnable: var bool) = +proc renderNimCodeOld(d: PDoc, n: PNode, dest: var Rope) = + ## this is a rather hacky way to get rid of the initial indentation + ## that the renderer currently produces: + # deadcode + var i = 0 + var body = n.lastSon + if body.len == 1 and body.kind == nkStmtList and + body.lastSon.kind == nkStmtList: + body = body.lastSon + for b in body: + if i > 0: dest.add "\n" + inc i + nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil) + +type RunnableState = enum + rsStart + rsComment + rsRunnable + rsDone + +proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: RunnableState): RunnableState = ##[ - previousIsRunnable: keep track of whether previous sibling was a runnableExample (true if 1st sibling though). - This is to ensure this works: + Simple state machine to tell whether we render runnableExamples and doc comments. + This is to ensure that we can interleave runnableExamples and doc comments freely; + the logic is easy to change but currently a doc comment following another doc comment + will not render, to avoid rendering in following case: + proc fn* = runnableExamples: discard ## d1 runnableExamples: discard ## d2 - ## d3 # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment - - It also works with: - proc fn* = - ## d0 - runnableExamples: discard - ## d1 - - etc + ## internal explanation # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment + discard # some code ]## - # xxx: checkme: owner check instead? this fails with the $nim/nimdoc/tester.nim test - # now that we're calling `genRecComment` only from here (to maintain correct order wrt runnableExample) - # if n.info.fileIndex != orig.info.fileIndex: return + case n.kind of nkCommentStmt: - if previousIsRunnable: + if state in {rsStart, rsRunnable}: dest.add genRecComment(d, n) - previousIsRunnable = false + return rsComment of nkCallKinds: if isRunnableExamples(n[0]) and - n.len >= 2 and n.lastSon.kind == nkStmtList: - previousIsRunnable = true - let rdoccmd = prepareExample(d, n) + n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}: + let (rdoccmd, code) = prepareExample(d, n) var msg = "Example:" if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd dispA(d.conf, dest, "\n

$1

\n", @@ -569,27 +587,71 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previous inc d.listingCounter let id = $d.listingCounter dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"]) - # this is a rather hacky way to get rid of the initial indentation - # that the renderer currently produces: - var i = 0 - var body = n.lastSon - if body.len == 1 and body.kind == nkStmtList and - body.lastSon.kind == nkStmtList: - body = body.lastSon - for b in body: - if i > 0: dest.add "\n" - inc i - nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil) + when true: + var dest2 = "" + renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex) + dest.add dest2 + else: + renderNimCodeOld(d, n, dest) dest.add(d.config.getOrDefault"doc.listing_end" % id) - else: previousIsRunnable = false + return rsRunnable + else: discard + return rsDone + # change this to `rsStart` if you want to keep generating doc comments + # and runnableExamples that occur after some code in routine - var previousIsRunnable2 = true - for i in 0.. + result = + ## foo + 3; + + proc someType*(): int = + ## foo + 3 +=> + ## foo + result = 3; + + so we normalize the results to get to the statement list containing the + (0 or more) doc comments and runnableExamples. + (even if using `result = n[bodyPos]`, you'd still to apply similar logic). + ]## + result = n[^1] + case result.kind + of nkSym: + result = n[^2] + case result.kind + of nkAsgn: + doAssert result[0].kind == nkSym + doAssert result.len == 2 + result = result[1] + else: # eg: nkStmtList + discard + else: + discard + +proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) = + var n = n + var state = rsStart + template fn(n2) = + state = getAllRunnableExamplesImpl(d, n2, dest, state) + case n.kind + of routineDefs: + n = n.getRoutineBody + case n.kind + of nkCommentStmt, nkCallKinds: fn(n) + else: + for i in 0.. conf.m.fileInfos[i.fileIndex.int32].lines.len: return "" + if i.line.int > num: return "" result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1] diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim new file mode 100644 index 0000000000..a784331635 --- /dev/null +++ b/compiler/renderverbatim.nim @@ -0,0 +1,79 @@ +import strutils +from xmltree import addEscaped + +import ast, options, msgs +import packages/docutils/highlite + +# import compiler/renderer +import renderer + +proc lastNodeRec(n: PNode): PNode = + result = n + while result.safeLen > 0: result = result[^1] + +proc isInIndentationBlock(src: string, indent: int): bool = + #[ + we stop at the first de-indentation; there's an inherent ambiguity with non + doc comments since they can have arbitrary indentation, so we just take the + practical route and require a runnableExamples to keep its code (including non + doc comments) to its indentation level. + ]# + for j in 0.. n[0].col` is part of runnableExamples + + let last = n.lastNodeRec.info + var info = first + var indent = info.col + let numLines = numLines(conf, info.fileIndex).uint16 + var lastNonemptyPos = 0 + for line in first.line..numLines: # bugfix, see `testNimDocTrailingExample` + info.line = line + let src = sourceLine(conf, info) + if line > last.line and not isInIndentationBlock(src, indent): + break + if line > first.line: result.add "\n" + if src.len > indent: + result.add src[indent..^1] + lastNonemptyPos = result.len + result.setLen lastNonemptyPos + +proc renderNimCode*(result: var string, code: string, isLatex = false) = + var toknizr: GeneralTokenizer + initGeneralTokenizer(toknizr, code) + var buf = "" + template append(kind, val) = + buf.setLen 0 + buf.addEscaped(val) + let class = tokenClassToStr[kind] + if isLatex: + result.addf "\\span$1{$2}" % [class, buf] + else: + result.addf "$2" % [class, buf] + + while true: + getNextToken(toknizr, langNim) + case toknizr.kind + of gtEof: break # End Of File (or string) + else: + # TODO: avoid alloc; maybe toOpenArray + append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1)) diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index 58272a9b67..15dd32ec7d 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -1,5 +1,6 @@ # Small program that runs the test cases for 'nim doc'. # To run this, cd to the git repo root, and run "nim c -r nimdoc/tester.nim". +# to change expected results (after carefully verifying everything), use -d:fixup import strutils, os diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 469dde0aee..6e83f718ff 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -129,6 +129,8 @@ function main() { title="aEnum(): untyped">aEnum
  • bEnum
  • +
  • fromUtilsGen
  • @@ -193,6 +195,15 @@ constructor. + + +
    template fromUtilsGen(): untyped
    +
    + +this should be shown in utils.html +

    Example:

    +
    assert 3*2 == 6
    ditto +
    diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index c848fc26a7..b49a777c8e 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -10,3 +10,4 @@ SomeType subdir/subdir_b/utils.html#SomeType utils: SomeType someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped +fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index aa29c0c0dc..420b609242 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -115,6 +115,9 @@ function main() {
  • B
  • +
  • Foo
  • @@ -149,8 +152,42 @@ function main() { title="baz[T](a, b: T): T">baz
  • buzz
  • +
  • fromUtils3
  • isValid
  • +
  • z1Foo
  • +
  • z2
  • +
  • z3
  • +
  • z4
  • +
  • z5
  • +
  • z6
  • +
  • z7
  • +
  • z8
  • +
  • z9
  • +
  • z10
  • +
  • z11
  • +
  • z12
  • +
  • z13
  • +
  • baz
  • +
  • z17
  • +
  • p1
  • @@ -162,19 +199,43 @@ function main() { +
  • + Iterators + +
  • Macros
  • Templates
  • @@ -187,13 +248,17 @@ function main() {

    This is the top level module.

    Example:

    -
    import
    -  subdir / subdir_b / utils
    -
    -doAssert bar(3, 4) == 7
    -foo(enumValueA, enumValueB)
    -for x in "xx":
    -  discard

    +
    import subdir / subdir_b / utils
    +doAssert bar(3, 4) == 7
    +foo(enumValueA, enumValueB)
    +# bug #11078
    +for x in "xx": discard
    top2 +

    Example:

    +
    discard "in top2"
    top2 after +

    Example:

    +
    discard "in top3"
    top3 after +

    Example:

    +
    assert 3*2 == 6

    Imports

    @@ -217,6 +282,14 @@ The enum A. The enum B. + + +
    Foo = enum
    +  enumValueA2
    +
    + + +
    @@ -294,6 +367,15 @@ This is deprecated without message. This is deprecated with a message. + + +
    proc fromUtils3() {...}{.raises: [], tags: [].}
    +
    + +came form utils but should be shown where fromUtilsGen is called +

    Example:

    +
    discard 1
    +
    proc isValid[T](x: T): bool
    @@ -301,6 +383,148 @@ This is deprecated with a message. + + +
    proc z1(): Foo {...}{.raises: [], tags: [].}
    +
    + +cz1 + +
    + +
    proc z2() {...}{.raises: [], tags: [].}
    +
    + +cz2 +

    Example:

    +
    discard "in cz2"
    + +
    + +
    proc z3() {...}{.raises: [], tags: [].}
    +
    + +cz3 + +
    + +
    proc z4() {...}{.raises: [], tags: [].}
    +
    + +cz4 + +
    + +
    proc z5(): int {...}{.raises: [], tags: [].}
    +
    + +cz5 + +
    + +
    proc z6(): int {...}{.raises: [], tags: [].}
    +
    + +cz6 + +
    + +
    proc z7(): int {...}{.raises: [], tags: [].}
    +
    + +cz7 + +
    + +
    proc z8(): int {...}{.raises: [], tags: [].}
    +
    + +cz8 + +
    + +
    proc z9() {...}{.raises: [], tags: [].}
    +
    + + +

    Example:

    +
    doAssert 1 + 1 == 2
    + +
    + +
    proc z10() {...}{.raises: [], tags: [].}
    +
    + + +

    Example: cmd: -d:foobar

    +
    discard 1
    cz10 + +
    + +
    proc z11() {...}{.raises: [], tags: [].}
    +
    + + +

    Example:

    +
    discard 1
    + +
    + +
    proc z12(): int {...}{.raises: [], tags: [].}
    +
    + + +

    Example:

    +
    discard 1
    + +
    + +
    proc z13() {...}{.raises: [], tags: [].}
    +
    + +cz13 +

    Example:

    +
    discard
    + +
    + +
    proc baz() {...}{.raises: [], tags: [].}
    +
    + + + +
    + +
    proc z17() {...}{.raises: [], tags: [].}
    +
    + +cz17 rest +

    Example:

    +
    discard 1
    rest + +
    + +
    proc p1() {...}{.raises: [], tags: [].}
    +
    + +cp1 +

    Example:

    +
    doAssert 1 == 1 # regular comments work here
    c4 +

    Example:

    +
    # c5 regular comments before 1st token work
    +# regular comment
    +#[
    +nested regular comment
    +]#
    +doAssert 2 == 2 # c8
    +## this is a non-nested doc comment
    +
    +##[
    +this is a nested doc comment
    +]##
    +discard "c9"
    +# also work after
    +
    @@ -315,6 +539,22 @@ My someFunc. Stuff in quotes + +
    +

    Iterators

    +
    + +
    iterator fromUtils1(): int {...}{.raises: [], tags: [].}
    +
    + + +

    Example:

    +
    # ok1
    +assert 1 == 1
    +# ok2
    + +
    +

    Macros

    @@ -325,18 +565,105 @@ My someFunc. Stuff in quotes + +
    macro z16()
    +
    + + +

    Example:

    +
    discard 1
    cz16 after +

    Example:

    +
    doAssert 2 == 1 + 1
    + +
    + +
    macro z18(): int
    +
    + +cz18 +

    Templates

    + +
    template fromUtils2()
    +
    + +ok3 + +
    + +
    template z6t(): int
    +
    + +cz6t + +
    template foo(a, b: SomeType)
    This does nothing +
    + +
    template myfn()
    +
    + + +

    Example:

    +
    import std/strutils
    +## issue #8871 preserve formatting
    +## line doc comment
    +# bar
    +doAssert "'foo" == "'foo"
    +##[
    +foo
    +bar
    +]##
    +
    +doAssert: not "foo".startsWith "ba"
    +block:
    +  discard 0xff # elu par cette crapule
    +# should be in
    should be still in + +
    + +
    template z14()
    +
    + +cz14 +

    Example:

    +
    discard
    + +
    + +
    template z15()
    +
    + +cz15 +

    Example:

    +
    discard
    +

    Example:

    +
    discard 3
    +

    Example:

    +
    discard 4
    ok5 ok5b +

    Example:

    +
    assert true
    in or out? + +
    + +
    template testNimDocTrailingExample()
    +
    + + +

    Example:

    +
    discard 2
    +
    diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index 106c9cb76c..76193de87d 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -2,13 +2,41 @@ C_A testproject.html#C_A testproject: C_A C_B testproject.html#C_B testproject: C_B C_C testproject.html#C_C testproject: C_C C_D testproject.html#C_D testproject: C_D -foo testproject.html#foo.t,SomeType,SomeType testproject: foo(a, b: SomeType) bar testproject.html#bar,T,T testproject: bar[T](a, b: T): T baz testproject.html#baz,T,T testproject: baz[T](a, b: T): T buzz testproject.html#buzz,T,T testproject: buzz[T](a, b: T): T -bar testproject.html#bar.m testproject: bar(): untyped aVariable testproject.html#aVariable testproject: aVariable A testproject.html#A testproject: A B testproject.html#B testproject: B someFunc testproject.html#someFunc testproject: someFunc() +fromUtils1 testproject.html#fromUtils1.i testproject: fromUtils1(): int +fromUtils2 testproject.html#fromUtils2.t testproject: fromUtils2() +fromUtils3 testproject.html#fromUtils3 testproject: fromUtils3() isValid testproject.html#isValid,T testproject: isValid[T](x: T): bool +enumValueA2 testproject.html#enumValueA2 Foo.enumValueA2 +Foo testproject.html#Foo testproject: Foo +z1 testproject.html#z1 testproject: z1(): Foo +z2 testproject.html#z2 testproject: z2() +z3 testproject.html#z3 testproject: z3() +z4 testproject.html#z4 testproject: z4() +z5 testproject.html#z5 testproject: z5(): int +z6 testproject.html#z6 testproject: z6(): int +z6t testproject.html#z6t.t testproject: z6t(): int +z7 testproject.html#z7 testproject: z7(): int +z8 testproject.html#z8 testproject: z8(): int +z9 testproject.html#z9 testproject: z9() +z10 testproject.html#z10 testproject: z10() +z11 testproject.html#z11 testproject: z11() +z12 testproject.html#z12 testproject: z12(): int +z13 testproject.html#z13 testproject: z13() +baz testproject.html#baz testproject: baz() +z17 testproject.html#z17 testproject: z17() +p1 testproject.html#p1 testproject: p1() +bar testproject.html#bar.m testproject: bar(): untyped +z16 testproject.html#z16.m testproject: z16() +z18 testproject.html#z18.m testproject: z18(): int +foo testproject.html#foo.t,SomeType,SomeType testproject: foo(a, b: SomeType) +myfn testproject.html#myfn.t testproject: myfn() +z14 testproject.html#z14.t testproject: z14() +z15 testproject.html#z15.t testproject: z15() +testNimDocTrailingExample testproject.html#testNimDocTrailingExample.t testproject: testNimDocTrailingExample() diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 18a7ca2c29..df118d8a67 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -97,6 +97,8 @@ function main() {
    baz:
    bEnum:
    +
    enumValueA2:
    enumValueB:
    +
    Foo:
    foo:
    +
    fromUtils1:
    +
    fromUtils2:
    +
    fromUtils3:
    +
    fromUtilsGen:
    isValid:
    +
    myfn:
    +
    p1:
    someFunc:
    +
    testNimDocTrailingExample:
    +
    z1:
    +
    z10:
    +
    z11:
    +
    z12:
    +
    z13:
    +
    z14:
    +
    z15:
    +
    z16:
    +
    z17:
    +
    z18:
    +
    z2:
    +
    z3:
    +
    z4:
    +
    z5:
    +
    z6:
    +
    z6t:
    +
    z7:
    +
    z8:
    +
    z9: