further progress on rst roles & directives (fix #17646) (#17659)

* further progress on rst roles & dir-s (fix #17646)

* fix documents according to the messages

* fix bug 17 from #17340

doc/manual_experimental.rst

@@ -330,7 +330,7 @@ Routines with the same type signature can be called differently if a parameter
 has different names. This does not need an `experimental` switch, but is an
 unstable feature.
 
-.. code-block::nim
+.. code-block:: Nim
   proc foo(x: int) =
     echo "Using x: ", x
   proc foo(y: int) =

lib/experimental/diff.nim

@@ -12,14 +12,14 @@
 ##
 ## A basic example of `diffInt` on 2 arrays of integers:
 ##
-## .. code::nim
+## .. code:: Nim
 ##
 ##   import experimental/diff
 ##   echo diffInt([0, 1, 2, 3, 4, 5, 6, 7, 8], [-1, 1, 2, 3, 4, 5, 666, 7, 42])
 ##
 ## Another short example of `diffText` to diff strings:
 ##
-## .. code::nim
+## .. code:: Nim
 ##
 ##   import experimental/diff
 ##   # 2 samples of text for testing (from "The Call of Cthulhu" by Lovecraft)

lib/packages/docutils/highlite.nim

@@ -13,7 +13,7 @@
 ##
 ## You can use this to build your own syntax highlighting, check this example:
 ##
-## .. code::nim
+## .. code:: Nim
 ##   let code = """for x in $int.high: echo x.ord mod 2 == 0"""
 ##   var toknizr: GeneralTokenizer
 ##   initGeneralTokenizer(toknizr, code)
@@ -34,7 +34,7 @@
 ##
 ## The proc `getSourceLanguage` can get the language `enum` from a string:
 ##
-## .. code::nim
+## .. code:: Nim
 ##   for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l)
 ##
 

lib/packages/docutils/rst.nim

@@ -912,6 +912,38 @@ template newLeaf(s: string): PRstNode = newRstLeaf(s)
 proc newLeaf(p: var RstParser): PRstNode =
   result = newLeaf(currentTok(p).symbol)
 
+proc validRefnamePunct(x: string): bool =
+  ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
+  x.len == 1 and x[0] in {'-', '_', '.', ':', '+'}
+
+func getRefnameIdx(p: RstParser, startIdx: int): int =
+  ## Gets last token index of a refname ("word" in RST terminology):
+  ##
+  ##   reference names are single words consisting of alphanumerics plus
+  ##   isolated (no two adjacent) internal hyphens, underscores, periods,
+  ##   colons and plus signs; no whitespace or other characters are allowed.
+  ##
+  ## Refnames are used for:
+  ## - reference names
+  ## - role names
+  ## - directive names
+  ## - footnote labels
+  ##
+  # TODO: use this func in all other relevant places
+  var j = startIdx
+  if p.tok[j].kind == tkWord:
+    inc j
+    while p.tok[j].kind == tkPunct and validRefnamePunct(p.tok[j].symbol) and
+        p.tok[j+1].kind == tkWord:
+      inc j, 2
+  result = j - 1
+
+func getRefname(p: RstParser, startIdx: int): (string, int) =
+  let lastIdx = getRefnameIdx(p, startIdx)
+  result[1] = lastIdx
+  for j in startIdx..lastIdx:
+    result[0].add p.tok[j].symbol
+
 proc getReferenceName(p: var RstParser, endStr: string): PRstNode =
   var res = newRstNode(rnInner)
   while true:
@@ -1011,7 +1043,10 @@ proc match(p: RstParser, start: int, expr: string): bool =
   var last = expr.len - 1
   while i <= last:
     case expr[i]
-    of 'w': result = p.tok[j].kind == tkWord
+    of 'w':
+      let lastIdx = getRefnameIdx(p, j)
+      result = lastIdx >= j
+      if result: j = lastIdx
     of ' ': result = p.tok[j].kind == tkWhite
     of 'i': result = p.tok[j].kind == tkIndent
     of 'I': result = p.tok[j].kind in {tkIndent, tkEof}
@@ -1058,7 +1093,7 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) =
 proc whichRole(p: RstParser, sym: string): RstNodeKind =
   result = whichRoleAux(sym)
   if result == rnUnknownRole:
-    rstMessage(p, mwUnsupportedLanguage, p.s.currRole)
+    rstMessage(p, mwUnsupportedLanguage, sym)
 
 proc toInlineCode(n: PRstNode, language: string): PRstNode =
   ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
@@ -1078,6 +1113,11 @@ proc toInlineCode(n: PRstNode, language: string): PRstNode =
   lb.add newLeaf(s)
   result.add lb
 
+proc toUnknownRole(n: PRstNode, roleName: string): PRstNode =
+  let newN = newRstNode(rnInner, n.sons)
+  let newSons = @[newN, newLeaf(roleName)]
+  result = newRstNode(rnUnknownRole, newSons)
+
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
   var newKind = n.kind
   var newSons = n.sons
@@ -1102,17 +1142,15 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
     result = newRstNode(newKind, newSons)
   elif match(p, p.idx, ":w:"):
     # a role:
-    let roleName = nextTok(p).symbol
+    let (roleName, lastIdx) = getRefname(p, p.idx+1)
     newKind = whichRole(p, roleName)
     if newKind == rnUnknownRole:
-      let newN = newRstNode(rnInner, n.sons)
-      newSons = @[newN, newLeaf(roleName)]
-      result = newRstNode(newKind, newSons)
+      result = n.toUnknownRole(roleName)
     elif newKind == rnInlineCode:
       result = n.toInlineCode(language=roleName)
     else:
       result = newRstNode(newKind, newSons)
-    inc p.idx, 3
+    p.idx = lastIdx + 2
   else:
     if p.s.currRoleKind == rnInlineCode:
       result = n.toInlineCode(language=p.s.currRole)
@@ -1139,10 +1177,6 @@ proc parseSmiley(p: var RstParser): PRstNode =
       result.text = val
       return
 
-proc validRefnamePunct(x: string): bool =
-  ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
-  x.len == 1 and x[0] in {'-', '_', '.', ':', '+'}
-
 proc isUrl(p: RstParser, i: int): bool =
   result = p.tok[i+1].symbol == ":" and p.tok[i+2].symbol == "//" and
     p.tok[i+3].kind == tkWord and
@@ -1373,14 +1407,18 @@ proc parseInline(p: var RstParser, father: PRstNode) =
       var n = newRstNode(rnInlineLiteral)
       parseUntil(p, n, "``", false)
       father.add(n)
-    elif match(p, p.idx, ":w:") and p.tok[p.idx+3].symbol == "`":
-      let roleName = nextTok(p).symbol
+    elif match(p, p.idx, ":w:") and
+        (var lastIdx = getRefnameIdx(p, p.idx + 1);
+         p.tok[lastIdx+2].symbol == "`"):
+      let (roleName, _) = getRefname(p, p.idx+1)
       let k = whichRole(p, roleName)
       var n = newRstNode(k)
-      inc p.idx, 3
+      p.idx = lastIdx + 2
       if k == rnInlineCode:
         n = n.toInlineCode(language=roleName)
       parseUntil(p, n, "`", false) # bug #17260
+      if k == rnUnknownRole:
+        n = n.toUnknownRole(roleName)
       father.add(n)
     elif isInlineMarkupStart(p, "`"):
       var n = newRstNode(rnInterpretedText)
@@ -1438,25 +1476,28 @@ proc parseInline(p: var RstParser, father: PRstNode) =
   else: discard
 
 proc getDirective(p: var RstParser): string =
-  if currentTok(p).kind == tkWhite and nextTok(p).kind == tkWord:
-    var j = p.idx
-    inc p.idx
-    result = currentTok(p).symbol
-    inc p.idx
-    while currentTok(p).kind in {tkWord, tkPunct, tkAdornment, tkOther}:
-      if currentTok(p).symbol == "::": break
-      result.add(currentTok(p).symbol)
-      inc p.idx
-    if currentTok(p).kind == tkWhite: inc p.idx
-    if currentTok(p).symbol == "::":
-      inc p.idx
-      if currentTok(p).kind == tkWhite: inc p.idx
-    else:
-      p.idx = j               # set back
-      result = ""             # error
-  else:
-    result = ""
-  result = result.toLowerAscii()
+  result = ""
+  if currentTok(p).kind == tkWhite:
+    let (name, lastIdx) = getRefname(p, p.idx + 1)
+    let afterIdx = lastIdx + 1
+    if name.len > 0:
+      if p.tok[afterIdx].symbol == "::":
+        result = name
+        p.idx = afterIdx + 1
+        if currentTok(p).kind == tkWhite:
+          inc p.idx
+        elif currentTok(p).kind != tkIndent:
+          rstMessage(p, mwRstStyle,
+              "whitespace or newline expected after directive " & name)
+        result = result.toLowerAscii()
+      elif p.tok[afterIdx].symbol == ":":
+        rstMessage(p, mwRstStyle,
+            "double colon :: may be missing at end of '" & name & "'",
+            p.tok[afterIdx].line, p.tok[afterIdx].col)
+      elif p.tok[afterIdx].kind == tkPunct and p.tok[afterIdx].symbol[0] == ':':
+        rstMessage(p, mwRstStyle,
+            "too many colons for a directive (should be ::)",
+            p.tok[afterIdx].line, p.tok[afterIdx].col)
 
 proc parseComment(p: var RstParser): PRstNode =
   case currentTok(p).kind
@@ -1711,7 +1752,8 @@ proc whichSection(p: RstParser): RstNodeKind =
       return rnCodeBlock
     elif currentTok(p).symbol == "::":
       return rnLiteralBlock
-    elif currentTok(p).symbol == ".." and predNL(p):
+    elif currentTok(p).symbol == ".."  and predNL(p) and
+       nextTok(p).kind in {tkWhite, tkIndent}:
      return rnDirective
   case currentTok(p).kind
   of tkAdornment:

lib/pure/asyncdispatch.nim

@@ -42,7 +42,7 @@
 ##
 ## Code to read some data from a socket may look something like this:
 ##
-##   .. code-block::nim
+##   .. code-block:: Nim
 ##      var future = socket.recv(100)
 ##      future.addCallback(
 ##        proc () =

lib/pure/asyncftpclient.nim

@@ -21,7 +21,7 @@
 ## In order to begin any sort of transfer of files you must first
 ## connect to an FTP server. You can do so with the `connect` procedure.
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##    import std/[asyncdispatch, asyncftpclient]
 ##    proc main() {.async.} =
 ##      var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
@@ -41,7 +41,7 @@
 ## working directory before you do so with the `pwd` procedure, you can also
 ## instead specify an absolute path.
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##    import std/[asyncdispatch, asyncftpclient]
 ##    proc main() {.async.} =
 ##      var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
@@ -62,7 +62,7 @@
 ## Procs that take an `onProgressChanged` callback will call this every
 ## `progressInterval` milliseconds.
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##    import std/[asyncdispatch, asyncftpclient]
 ##
 ##    proc onProgressChanged(total, progress: BiggestInt,

lib/pure/asynchttpserver.nim

@@ -108,7 +108,7 @@ proc respond*(req: Request, code: HttpCode, content: string,
   ##
   ## Example:
   ##
-  ## .. code-block::nim
+  ## .. code-block:: Nim
   ##    import std/json
   ##    proc handler(req: Request) {.async.} =
   ##      if req.url.path == "/hello-world":

lib/pure/asyncnet.nim

@@ -65,7 +65,7 @@
 ##
 ## The following example demonstrates a simple chat server.
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##
 ##   import std/[asyncnet, asyncdispatch]
 ##

lib/pure/base64.nim

@@ -23,14 +23,14 @@
 ## Encoding data
 ## -------------
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##    import std/base64
 ##    let encoded = encode("Hello World")
 ##    assert encoded == "SGVsbG8gV29ybGQ="
 ##
 ## Apart from strings you can also encode lists of integers or characters:
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##    import std/base64
 ##    let encodedInts = encode([1,2,3])
 ##    assert encodedInts == "AQID"
@@ -41,7 +41,7 @@
 ## Decoding data
 ## -------------
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##    import std/base64
 ##    let decoded = decode("SGVsbG8gV29ybGQ=")
 ##    assert decoded == "Hello World"
@@ -49,7 +49,7 @@
 ## URL Safe Base64
 ## ---------------
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##    import std/base64
 ##    doAssert encode("c\xf7>", safe = true) == "Y_c-"
 ##    doAssert encode("c\xf7>", safe = false) == "Y/c+"

lib/pure/dynlib.nim

@@ -22,7 +22,7 @@
 ## If the library fails to load or the function 'greet' is not found,
 ## it quits with a failure error code.
 ##
-## .. code-block::nim
+## .. code-block:: Nim
 ##
 ##   import std/dynlib
 ##

lib/system/iterators.nim

@@ -304,7 +304,7 @@ iterator fieldPairs*[T: tuple|object](x: T): tuple[key: string, val: RootObj] {.
   ## picking the appropriate code to a secondary proc which you overload for
   ## each field type and pass the `value` to.
   ##
-  ## .. warning::: This really transforms the 'for' and unrolls the loop. The
+  ## .. warning:: This really transforms the 'for' and unrolls the loop. The
   ##   current implementation also has a bug that affects symbol binding in the
   ##   loop body.
   runnableExamples:

tests/stdlib/trstgen.nim

@@ -136,6 +136,25 @@ suite "YAML syntax highlighting":
   <span class="StringLit">not numbers</span><span class="Punctuation">:</span> <span class="Punctuation">[</span> <span class="StringLit">42e</span><span class="Punctuation">,</span> <span class="StringLit">0023</span><span class="Punctuation">,</span> <span class="StringLit">+32.37</span><span class="Punctuation">,</span> <span class="StringLit">8 ball</span><span class="Punctuation">]</span>
 <span class="Punctuation">}</span></pre>"""
 
+  test "Directives: warnings":
+    let input = dedent"""
+      .. non-existant-warning: Paragraph.
+
+      .. another.wrong:warning::: Paragraph.
+      """
+    var warnings = new seq[string]
+    let output = input.toHtml(warnings=warnings)
+    check output == ""
+    doAssert warnings[].len == 2
+    check "(1, 24) Warning: RST style:" in warnings[0]
+    check "double colon :: may be missing at end of 'non-existant-warning'" in warnings[0]
+    check "(3, 25) Warning: RST style:" in warnings[1]
+    check "RST style: too many colons for a directive (should be ::)" in warnings[1]
+
+  test "not a directive":
+    let input = "..warning:: I am not a warning."
+    check input.toHtml == input
+
   test "Anchors, Aliases, Tags":
     let input = """.. code-block:: yaml
     --- !!map
@@ -1403,6 +1422,23 @@ Test1
     check """`3`:sup:\ He is an isotope of helium.""".toHtml == expected
     check """`3`:superscript:\ He is an isotope of helium.""".toHtml == expected
 
+  test "Roles: warnings":
+    let input = dedent"""
+      See function :py:func:`spam`.
+
+      See also `egg`:py:class:.
+      """
+    var warnings = new seq[string]
+    let output = input.toHtml(warnings=warnings)
+    doAssert warnings[].len == 2
+    check "(1, 14) Warning: " in warnings[0]
+    check "language 'py:func' not supported" in warnings[0]
+    check "(3, 15) Warning: " in warnings[1]
+    check "language 'py:class' not supported" in warnings[1]
+    check("""<p>See function <span class="py:func">spam</span>.</p>""" & "\n" &
+          """<p>See also <span class="py:class">egg</span>. </p>""" & "\n" ==
+          output)
+
   test "(not) Roles: check escaping 1":
     let expected = """See :subscript:<tt class="docutils literal">""" &
                    """<span class="pre">""" & id"some" & " " & id"text" &