From 2e71bd50b26e6165121a3076f7dc70175e9f1f55 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 16 Jan 2016 21:40:54 +0100 Subject: [PATCH] implements multi-line-comments; pounds are stripped from the doc comments --- compiler/docgen.nim | 4 +- compiler/lexer.nim | 87 +++++++++++++++++++++++++----- compiler/parser.nim | 7 +-- compiler/renderer.nim | 19 ++----- doc/manual/lexing.txt | 38 +++++++++++++ lib/packages/docutils/highlite.nim | 17 +++++- todo.txt | 2 +- 7 files changed, 137 insertions(+), 37 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 952366f93a..8555ec4f05 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -149,10 +149,10 @@ proc ropeFormatNamedVars(frmt: FormatStr, varnames: openArray[string], proc genComment(d: PDoc, n: PNode): string = result = "" var dummyHasToc: bool - if n.comment != nil and startsWith(n.comment, "##"): + if n.comment != nil: renderRstToOut(d[], parseRst(n.comment, toFilename(n.info), toLinenumber(n.info), toColumn(n.info), - dummyHasToc, d.options + {roSkipPounds}), result) + dummyHasToc, d.options), result) proc genRecComment(d: PDoc, n: PNode): Rope = if n == nil: return nil diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 9a69ede3e5..70a555e155 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -769,24 +769,78 @@ proc getOperator(L: var TLexer, tok: var TToken) = if buf[pos] in {CR, LF, nimlexbase.EndOfFile}: tok.strongSpaceB = -1 +proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; + isDoc: bool) = + var pos = start + var buf = L.buf + while buf[pos] == '[': + inc pos + let brackets = pos - start + var toStrip = 0 + # detect the amount of indentation: + if isDoc: + toStrip = getColNumber(L, pos) + while buf[pos] == ' ': inc pos + if buf[pos] in {CR, LF}: + pos = handleCRLF(L, pos) + buf = L.buf + toStrip = 0 + while buf[pos] == ' ': + inc pos + inc toStrip + while true: + case buf[pos] + of ']': + var p = pos + while buf[p] == ']': + inc p + if p-pos == brackets: + pos = p + break + else: + if isDoc: tok.literal.add ']' + inc pos + of '\t': + lexMessagePos(L, errTabulatorsAreNotAllowed, pos) + inc(pos) + if isDoc: tok.literal.add '\t' + of CR, LF: + pos = handleCRLF(L, pos) + buf = L.buf + # strip leading whitespace: + if isDoc: + tok.literal.add "\n" + inc tok.iNumber + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c + of nimlexbase.EndOfFile: + lexMessagePos(L, errGenerated, pos, "end of multiline comment expected") + break + else: + if isDoc: tok.literal.add buf[pos] + inc(pos) + L.bufpos = pos + proc scanComment(L: var TLexer, tok: var TToken) = var pos = L.bufpos var buf = L.buf - when not defined(nimfix): - assert buf[pos+1] == '#' - if buf[pos+2] == '[': - if buf[pos+3] == ']': - # ##[] is the (rather complex) "cursor token" for idetools - tok.tokType = tkComment - tok.literal = "[]" - inc(L.bufpos, 4) - return - else: - lexMessagePos(L, warnDeprecated, pos, "use '## [' instead; '##['") - tok.tokType = tkComment # iNumber contains the number of '\n' in the token tok.iNumber = 0 + when not defined(nimfix): + assert buf[pos+1] == '#' + if buf[pos+2] == '[': + skipMultiLineComment(L, tok, pos+2, true) + return + inc(pos, 2) + + var toStrip = 0 + while buf[pos] == ' ': + inc pos + inc toStrip + when defined(nimfix): var col = getColNumber(L, pos) while true: @@ -820,6 +874,12 @@ proc scanComment(L: var TLexer, tok: var TToken) = if doContinue(): tok.literal.add "\n" when defined(nimfix): col = indent + else: + inc(pos, 2) + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c inc tok.iNumber else: if buf[pos] > ' ': @@ -863,7 +923,8 @@ proc skip(L: var TLexer, tok: var TToken) = # do not skip documentation comment: if buf[pos+1] == '#': break if buf[pos+1] == '[': - lexMessagePos(L, warnDeprecated, pos, "use '# [' instead; '#['") + skipMultiLineComment(L, tok, pos+1, false) + return while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) else: break # EndOfFile also leaves the loop diff --git a/compiler/parser.nim b/compiler/parser.nim index eb0ef071d6..c4681a5cdb 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -112,12 +112,7 @@ proc rawSkipComment(p: var TParser, node: PNode) = if p.tok.tokType == tkComment: if node != nil: if node.comment == nil: node.comment = "" - if p.tok.literal == "[]": - node.flags.incl nfIsCursor - #echo "parser: " - #debug node - else: - add(node.comment, p.tok.literal) + add(node.comment, p.tok.literal) else: parMessage(p, errInternal, "skipComment") getTok(p) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index c4d1222c4a..8e4aa18314 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -167,33 +167,24 @@ proc makeNimString(s: string): string = proc putComment(g: var TSrcGen, s: string) = if s.isNil: return var i = 0 - var comIndent = 1 var isCode = (len(s) >= 2) and (s[1] != ' ') var ind = g.lineLen - var com = "" + var com = "## " while true: case s[i] of '\0': break of '\x0D': put(g, tkComment, com) - com = "" + com = "## " inc(i) if s[i] == '\x0A': inc(i) optNL(g, ind) of '\x0A': put(g, tkComment, com) - com = "" + com = "## " inc(i) optNL(g, ind) - of '#': - add(com, s[i]) - inc(i) - comIndent = 0 - while s[i] == ' ': - add(com, s[i]) - inc(i) - inc(comIndent) of ' ', '\x09': add(com, s[i]) inc(i) @@ -206,7 +197,7 @@ proc putComment(g: var TSrcGen, s: string) = if not isCode and (g.lineLen + (j - i) > MaxLineLen): put(g, tkComment, com) optNL(g, ind) - com = '#' & spaces(comIndent) + com = "## " while s[i] > ' ': add(com, s[i]) inc(i) @@ -283,7 +274,7 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool = result = false if n.comment != nil: result = (renderNoComments notin g.flags) or - (renderDocComments in g.flags) and startsWith(n.comment, "##") + (renderDocComments in g.flags) proc gcom(g: var TSrcGen, n: PNode) = assert(n != nil) diff --git a/doc/manual/lexing.txt b/doc/manual/lexing.txt index 7f81ab4223..a5c3dfd6de 100644 --- a/doc/manual/lexing.txt +++ b/doc/manual/lexing.txt @@ -69,6 +69,44 @@ Documentation comments are tokens; they are only allowed at certain places in the input file as they belong to the syntax tree! +Multiline comments +------------------ + +Starting with version 0.13.0 of the language Nim supports multiline comments. +They look like: + +.. code-block:: nim + #[Comment here. + Multiple lines + are not a problem.] + +.. code-block:: nim + #[[comment here]] + +.. code-block:: nim + #[[[comment here]]] + +.. code-block:: nim + #[[[[comment here]]]] + +The number of opening brackets must match the number of closing brackets +but they do not nest in the traditional sense: + +.. code-block:: nim + # Does not comment out 'p' properly: + #[ + proc p[T](x: T) = discard + ] + +Multiline documentation comments look like: + +.. code-block:: nim + proc foo = + ##[Long documentation comment + here. + ] + + Identifiers & Keywords ---------------------- diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 640b8cd5a4..570465e141 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -173,7 +173,22 @@ proc nimNextToken(g: var GeneralTokenizer) = while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) of '#': g.kind = gtComment - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) + inc(pos) + if g.buf[pos] == '#': inc(pos) + if g.buf[pos] == '[': + g.kind = gtLongComment + var brackets = 0 + while g.buf[pos] == '[': + inc(pos) + inc(brackets) + while g.buf[pos] != '\0': + if g.buf[pos] == ']': + var q = pos + while g.buf[pos] == ']': inc(pos) + if pos-q == brackets: break + inc(pos) + else: + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': var id = "" while g.buf[pos] in SymChars + {'_'}: diff --git a/todo.txt b/todo.txt index ca8d607626..6c1c602f53 100644 --- a/todo.txt +++ b/todo.txt @@ -10,7 +10,6 @@ essential for 1.0 - make '--implicitStatic:on' the default; then we can also clean up the 'static[T]' mess in the compiler! -- add "all threads are blocked" detection to 'spawn' - Deprecate ``immediate`` for templates and macros - document NimMain and check whether it works for threading - ``not`` or ``~`` for the effects system @@ -19,6 +18,7 @@ essential for 1.0 Not critical for 1.0 ==================== +- add "all threads are blocked" detection to 'spawn' - figure out why C++ bootstrapping is so much slower - The bitwise 'not' operator cold be renamed to 'bnot' to prevent 'not 4 == 5' from compiling. -> requires 'mixin' annotation for procs!