Improve Markdown code blocks & start moving docs to Markdown style (#19954)

- add additional parameters parsing (other implementations will just
  ignore them). E.g. if in RST we have:

  .. code:: nim
     :test: "nim c $1"

     ...

  then in Markdown that will be:

  ```nim test="nim c $1"
  ...
  ```

- implement Markdown interpretation of additional indentation which is
  less than 4 spaces (>=4 spaces is a code block but it's not
implemented yet). RST interpretes it as quoted block, for Markdown it's
just normal paragraphs.
- add separate `md2html` and `md2tex` commands. This is to separate
  Markdown behavior in cases when it diverges w.r.t. RST significantly —
most conspicously like in the case of additional indentation above, and
also currently the contradicting inline rule of Markdown is also turned
on only in `md2html` and `md2tex`. **Rationale:** mixing Markdown and
RST arbitrarily is a way to nowhere, we need to provide a way to fix the
particular behavior. Note that still all commands have **both** Markdown
and RST features **enabled**. In this PR `*.nim` files can be processed
only in Markdown mode, while `md2html` is for `*.md` files and
`rst2html` for `*.rst` files.
- rename `*.rst` files to `.*md` as our current default behavior is
  already Markdown-ish
- convert code blocks in `docgen.rst` to Markdown style as an example.
  Other code blocks will be converted in the follow-up PRs
- fix indentation inside Markdown code blocks — additional indentation
  is preserved there
- allow more than 3 backticks open/close blocks (tildas \~ are still not
  allowed to avoid conflict with RST adornment headings) see also
https://github.com/nim-lang/RFCs/issues/355
- better error messages
- (other) fix a bug that admonitions cannot be used in sandbox mode; fix
  annoying warning on line 2711

compiler/commands.nim

@@ -449,6 +449,8 @@ proc parseCommand*(command: string): Command =
   of "doc2", "doc": cmdDoc
   of "doc2tex": cmdDoc2tex
   of "rst2html": cmdRst2html
+  of "md2tex": cmdMd2tex
+  of "md2html": cmdMd2html
   of "rst2tex": cmdRst2tex
   of "jsondoc0": cmdJsondoc0
   of "jsondoc2", "jsondoc": cmdJsondoc
@@ -480,7 +482,8 @@ proc setCommandEarly*(conf: ConfigRef, command: string) =
   # command early customizations
   # must be handled here to honor subsequent `--hint:x:on|off`
   case conf.cmd
-  of cmdRst2html, cmdRst2tex: # xxx see whether to add others: cmdGendepend, etc.
+  of cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex:
+      # xxx see whether to add others: cmdGendepend, etc.
     conf.foreignPackageNotes = {hintSuccessX}
   else:
     conf.foreignPackageNotes = foreignPackageNotesDefault

compiler/docgen.nim

@@ -88,7 +88,7 @@ type
     jEntriesFinal: JsonNode    # final JSON after RST pass 2 and rendering
     types: TStrTable
     sharedState: PRstSharedState
-    isPureRst: bool
+    standaloneDoc: bool
     conf*: ConfigRef
     cache*: IdentCache
     exampleCounter: int
@@ -230,6 +230,7 @@ template declareClosures =
     case msgKind
     of meCannotOpenFile: k = errCannotOpenFile
     of meExpected: k = errXExpected
+    of meMissingClosing: k = errRstMissingClosing
     of meGridTableNotImplemented: k = errRstGridTableNotImplemented
     of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable
     of meIllformedTable: k = errRstIllformedTable
@@ -276,16 +277,18 @@ proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
 
 proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
                     outExt: string = HtmlExt, module: PSym = nil,
-                    isPureRst = false): PDoc =
+                    standaloneDoc = false, preferMarkdown = true): PDoc =
   declareClosures()
   new(result)
   result.module = module
   result.conf = conf
   result.cache = cache
   result.outDir = conf.outDir.string
-  result.isPureRst = isPureRst
-  var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roSandboxDisabled}
-  if not isPureRst: options.incl roNimFile
+  result.standaloneDoc = standaloneDoc
+  var options= {roSupportRawDirective, roSupportMarkdown, roSandboxDisabled}
+  if preferMarkdown:
+    options.incl roPreferMarkdown
+  if not standaloneDoc: options.incl roNimFile
   result.sharedState = newRstSharedState(
       options, filename.string,
       docgenFindFile, compilerMsgHandler)
@@ -333,7 +336,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
       # Make sure the destination directory exists
       createDir(outp.splitFile.dir)
       # Include the current file if we're parsing a nim file
-      let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")]
+      let importStmt = if d.standaloneDoc: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")]
       writeFile(outp, importStmt & content)
 
       proc interpSnippetCmd(cmd: string): string =
@@ -1512,7 +1515,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
         "\\\\\\vspace{0.5em}\\large $1", [esc(d.target, d.meta[metaSubtitle])])
 
   var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
-  let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd:
+  let bodyname = if d.hasToc and not d.standaloneDoc and not d.conf.isLatexCmd:
                    groupsection.setLen 0
                    "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"
@@ -1626,9 +1629,11 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
   generateIndex(d)
 
 proc commandRstAux(cache: IdentCache, conf: ConfigRef;
-                   filename: AbsoluteFile, outExt: string) =
+                   filename: AbsoluteFile, outExt: string,
+                   preferMarkdown: bool) =
   var filen = addFileExt(filename, "txt")
-  var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true)
+  var d = newDocumentor(filen, cache, conf, outExt, standaloneDoc = true,
+                        preferMarkdown = preferMarkdown)
   let rst = parseRst(readFile(filen.string),
                      line=LineRstInit, column=ColRstInit,
                      conf, d.sharedState)
@@ -1637,11 +1642,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef;
   writeOutput(d)
   generateIndex(d)
 
-proc commandRst2Html*(cache: IdentCache, conf: ConfigRef) =
-  commandRstAux(cache, conf, conf.projectFull, HtmlExt)
+proc commandRst2Html*(cache: IdentCache, conf: ConfigRef,
+                      preferMarkdown=false) =
+  commandRstAux(cache, conf, conf.projectFull, HtmlExt, preferMarkdown)
 
-proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) =
-  commandRstAux(cache, conf, conf.projectFull, TexExt)
+proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef,
+                     preferMarkdown=false) =
+  commandRstAux(cache, conf, conf.projectFull, TexExt, preferMarkdown)
 
 proc commandJson*(cache: IdentCache, conf: ConfigRef) =
   ## implementation of a deprecated jsondoc0 command

compiler/lineinfos.nim

@@ -32,6 +32,7 @@ type
     # non-fatal errors
     errIllFormedAstX, errCannotOpenFile,
     errXExpected,
+    errRstMissingClosing,
     errRstGridTableNotImplemented,
     errRstMarkdownIllformedTable,
     errRstIllformedTable,
@@ -105,6 +106,7 @@ const
     errIllFormedAstX: "illformed AST: $1",
     errCannotOpenFile: "cannot open '$1'",
     errXExpected: "'$1' expected",
+    errRstMissingClosing: "$1",
     errRstGridTableNotImplemented: "grid table is not implemented",
     errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table",
     errRstIllformedTable: "Illformed table: $1",

compiler/main.nim

@@ -276,7 +276,8 @@ proc mainCommand*(graph: ModuleGraph) =
     var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
               else: conf.projectPath
     doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
-    if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex}: ret = ret / htmldocsDir
+    if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex}:
+      ret = ret / htmldocsDir
     conf.outDir = ret
 
   ## process all commands
@@ -302,7 +303,7 @@ proc mainCommand*(graph: ModuleGraph) =
       commandDoc2(graph, HtmlExt)
       if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
         commandBuildIndex(conf, $conf.outDir)
-  of cmdRst2html:
+  of cmdRst2html, cmdMd2html:
     # XXX: why are warnings disabled by default for rst2html and rst2tex?
     for warn in rstWarnings:
       conf.setNoteDefaults(warn, true)
@@ -311,16 +312,16 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
       loadConfigs(DocConfig, cache, conf, graph.idgen)
-      commandRst2Html(cache, conf)
-  of cmdRst2tex, cmdDoc2tex:
+      commandRst2Html(cache, conf, preferMarkdown = (conf.cmd == cmdMd2html))
+  of cmdRst2tex, cmdMd2tex, cmdDoc2tex:
     for warn in rstWarnings:
       conf.setNoteDefaults(warn, true)
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
-      if conf.cmd == cmdRst2tex:
+      if conf.cmd in {cmdRst2tex, cmdMd2tex}:
         loadConfigs(DocTexConfig, cache, conf, graph.idgen)
-        commandRst2TeX(cache, conf)
+        commandRst2TeX(cache, conf, preferMarkdown = (conf.cmd == cmdMd2tex))
       else:
         docLikeCmd commandDoc2(graph, TexExt)
   of cmdJsondoc0: docLikeCmd commandJson(cache, conf)

compiler/nim.nim

@@ -122,7 +122,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
         # `The parameter is incorrect`
       let cmd = cmdPrefix & output.quoteShell & ' ' & conf.arguments
       execExternalProgram(conf, cmd.strip(leading=false,trailing=true))
-    of cmdDocLike, cmdRst2html, cmdRst2tex: # bugfix(cmdRst2tex was missing)
+    of cmdDocLike, cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex: # bugfix(cmdRst2tex was missing)
       if conf.arguments.len > 0:
         # reserved for future use
         rawMessage(conf, errGenerated, "'$1 cannot handle arguments" % [$conf.cmd])

compiler/options.nim

@@ -153,6 +153,8 @@ type
     cmdDoc2tex  # convert .nim doc comments to LaTeX
     cmdRst2html # convert a reStructuredText file to HTML
     cmdRst2tex # convert a reStructuredText file to TeX
+    cmdMd2html # convert a Markdown file to HTML
+    cmdMd2tex # convert a Markdown file to TeX
     cmdJsondoc0
     cmdJsondoc
     cmdCtags

doc/backends.md

@@ -10,7 +10,8 @@
 .. no syntax highlighting here by default:
 
 .. contents::
-  "Heresy grows from idleness." -- Unknown.
+
+> "Heresy grows from idleness." -- Unknown.
 
 
 Introduction

doc/contributing.md

@@ -581,7 +581,7 @@ Code reviews
 
 
 
-.. include:: docstyle.rst
+.. include:: docstyle.md
 
 
 Evolving the stdlib

doc/docgen.md

@@ -35,14 +35,13 @@ Quick start
 
 Generate HTML documentation for a file:
 
-.. code:: cmd
-
+  ```cmd
   nim doc <filename>.nim
+  ```
 
 Generate HTML documentation for a whole project:
 
-.. code:: cmd
-
+  ```cmd
   # delete any htmldocs/*.idx file before starting
   nim doc --project --index:on --git.url:<url> --git.commit:<tag> --outdir:htmldocs <main_filename>.nim
   # this will generate html files, a theindex.html index, css and js under `htmldocs`
@@ -54,7 +53,7 @@ Generate HTML documentation for a whole project:
   # or `$nimcache/htmldocs` with `--usenimcache` which avoids clobbering your sources;
   # and likewise without `--project`.
   # Adding `-r` will open in a browser directly.
-
+  ```
 
 Documentation Comments
 ----------------------
@@ -120,8 +119,8 @@ Example of Nim file input
 The following examples will generate documentation for this sample
 *Nim* module, aptly named ``doc/docgen_sample.nim``:
 
-.. code:: nim
-   :file: docgen_sample.nim
+   ```nim file=docgen_sample.nim
+   ```
 
 All the below commands save their output to ``htmldocs`` directory relative to
 the directory of file;
@@ -137,9 +136,9 @@ optionally, an index file.
 
 The `doc`:option: command:
 
-.. code:: cmd
-
+  ```cmd
   nim doc docgen_sample.nim
+  ```
 
 Partial Output::
   ...
@@ -159,8 +158,7 @@ HTML -> PDF conversion).
 
 The `doc2tex`:option: command:
 
-.. code:: cmd
-
+  ```cmd
   nim doc2tex docgen_sample.nim
   cd htmldocs
   xelatex docgen_sample.tex
@@ -169,6 +167,7 @@ The `doc2tex`:option: command:
   # large documents) to get all labels generated.
   # That depends on this warning in the end of `xelatex` output:
   #   LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right.
+  ```
 
 The output is ``docgen_sample.pdf``.
 
@@ -183,9 +182,9 @@ Note that this tool is built off of the `doc`:option: command
 
 The `jsondoc`:option: command:
 
-.. code:: cmd
-
+  ```cmd
   nim jsondoc docgen_sample.nim
+  ```
 
 Output::
   {
@@ -209,9 +208,9 @@ renamed to `jsondoc0`:option:.
 
 The `jsondoc0`:option: command:
 
-.. code:: cmd
-
+  ```cmd
   nim jsondoc0 docgen_sample.nim
+  ```
 
 Output::
   [
@@ -249,9 +248,9 @@ the anchor [*]_ of Nim symbol that corresponds to link text.
 
 If you have a constant:
 
-.. code:: Nim
-
+   ```Nim
    const pi* = 3.14
+   ```
 
 then it should be referenced in one of the 2 forms:
 
@@ -262,9 +261,9 @@ B. qualified (with symbol kind specification)::
 
 For routine kinds there are more options. Consider this definition:
 
-.. code:: Nim
-
+   ```Nim
    proc foo*(a: int, b: float): string
+   ```
 
 Generally following syntax is allowed for referencing `foo`:
 
@@ -352,11 +351,11 @@ recognized fine::
    (without parameter names, see form A.2 above).
    E.g. for this signature:
 
-   .. code:: Nim
-
+      ```Nim
       proc binarySearch*[T, K](a: openArray[T]; key: K;
                                cmp: proc (x: T; y: K): int {.closure.}): int
                                           ~~    ~~   ~~~~~
+      ```
 
    you cannot use names underlined by `~~` so it must be referenced with
    ``cmp: proc(T, K)``. Hence these forms are valid::
@@ -379,10 +378,10 @@ recognized fine::
 .. Note:: A bit special case is operators
    (as their signature is also defined with `\``):
 
-   .. code:: Nim
-
+      ```Nim
       func `$`(x: MyType): string
       func `[]`*[T](x: openArray[T]): T
+      ```
 
    A short form works without additional backticks::
 
@@ -412,9 +411,9 @@ Related Options
 Project switch
 --------------
 
-.. code:: cmd
-
+  ```cmd
   nim doc --project filename.nim
+  ```
 
 This will recursively generate documentation of all Nim modules imported
 into the input module that belong to the Nimble package that ``filename.nim``
@@ -425,9 +424,9 @@ also be generated.
 Index switch
 ------------
 
-.. code:: cmd
-
+  ```cmd
   nim doc --index:on filename.nim
+  ```
 
 This will generate an index of all the exported symbols in the input Nim
 module, and put it into a neighboring file with the extension of ``.idx``. The
@@ -443,9 +442,9 @@ file.
 See source switch
 -----------------
 
-.. code:: cmd
-
+  ```cmd
   nim doc --git.url:<url> filename.nim
+  ```
 
 With the `git.url`:option: switch the *See source* hyperlink will appear below each
 documented item in your source code pointing to the implementation of that
@@ -490,9 +489,9 @@ supports highlighting of a few other languages supported by the
 
 Usage:
 
-.. code:: cmd
-
+  ```cmd
   nim rst2html docgen.rst
+  ```
 
 Output::
   You're reading it!
@@ -528,7 +527,7 @@ HTML file, most browsers will go to the first one. To differentiate the rest,
 you will need to use the complex name. A complex name for a callable type is
 made up of several parts:
 
-    (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\*
+  (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\*
 
 The first thing to note is that all callable types have at least a comma, even
 if they don't have any parameters. If there are parameters, they are

doc/intern.md

@@ -10,7 +10,7 @@
 .. include:: rstcommon.rst
 .. contents::
 
-  "Abstraction is layering ignorance on top of reality." -- Richard Gabriel
+> "Abstraction is layering ignorance on top of reality." -- Richard Gabriel
 
 
 Directory structure
@@ -276,8 +276,8 @@ and `exitingDebugSection()`:nim:.
 #. Compile the temp compiler with `--debugger:native -d:nimDebugUtils`:option:
 #. Set your desired breakpoints or watchpoints.
 #. Configure your debugger:
-  * GDB: execute `source tools/compiler.gdb` at startup
-  * LLDB execute `command source tools/compiler.lldb` at startup
+   * GDB: execute `source tools/compiler.gdb` at startup
+   * LLDB execute `command source tools/compiler.lldb` at startup
 #. Use one of the scoping helpers like so:
 
 .. code-block:: nim

doc/manual.md

@@ -10,9 +10,9 @@ Nim Manual
 .. contents::
 
 
-  "Complexity" seems to be a lot like "energy": you can transfer it from the
-  end-user to one/some of the other players, but the total amount seems to remain
-  pretty much constant for a given task. -- Ran
+> "Complexity" seems to be a lot like "energy": you can transfer it from the
+> end-user to one/some of the other players, but the total amount seems to remain
+> pretty much constant for a given task. -- Ran
 
 
 About this document
@@ -4025,7 +4025,7 @@ In the standard library every name of a routine that returns a `var` type
 starts with the prefix `m` per convention.
 
 
-.. include:: manual/var_t_return.rst
+.. include:: manual/var_t_return.md
 
 Future directions
 ~~~~~~~~~~~~~~~~~

doc/manual_experimental.md

@@ -505,7 +505,7 @@ The compiler ensures that every code path initializes variables which contain
 non-nilable pointers. The details of this analysis are still to be specified
 here.
 
-.. include:: manual_experimental_strictnotnil.rst
+.. include:: manual_experimental_strictnotnil.md
 
 
 Aliasing restrictions in parameter passing

doc/mm.md

@@ -11,7 +11,7 @@ Nim's Memory Management
 ..
 
 
-  "The road to hell is paved with good intentions."
+> "The road to hell is paved with good intentions."
 
 
 Multi-paradigm Memory Management Strategies

doc/nimc.md

@@ -11,9 +11,9 @@
 
 ..
 
-  "Look at you, hacker. A pathetic creature of meat and bone, panting and
-  sweating as you run through my corridors. How can you challenge a perfect,
-  immortal machine?"
+> "Look at you, hacker. A pathetic creature of meat and bone, panting and
+> sweating as you run through my corridors. How can you challenge a perfect,
+> immortal machine?"
 
 
 Introduction

doc/nimgrep.md

@@ -54,7 +54,7 @@ All examples below use default PCRE Regex patterns:
     nimgrep --excludeDir:'^\.git$' --excludeDir:'^\.hg$' --excludeDir:'^\.svn$'
     # short: --ed:'^\.git$' --ed:'^\.hg$' --ed:'^\.svn$'
 
-+ To search only in paths containing the `tests` sub-directory recursively::
++ To search only in paths containing the `tests` sub-directory recursively:
 
   .. code:: cmd
     nimgrep --recursive --includeDir:'(^|/)tests($|/)'

doc/overview.md

@@ -5,5 +5,5 @@ Nim Documentation Overview
 :Author: Andreas Rumpf
 :Version: |nimversion|
 
-.. include:: docs.rst
+.. include:: docs.md
 

doc/tut2.md

@@ -13,7 +13,7 @@ Nim Tutorial (Part II)
 Introduction
 ============
 
-  "Repetition renders the ridiculous reasonable." -- Norman Wildberger
+> "Repetition renders the ridiculous reasonable." -- Norman Wildberger
 
 This document is a tutorial for the advanced constructs of the *Nim*
 programming language. **Note that this document is somewhat obsolete as the**

doc/tut3.md

@@ -13,7 +13,7 @@ Nim Tutorial (Part III)
 Introduction
 ============
 
-  "With Great Power Comes Great Responsibility." -- Spider Man's Uncle
+>  "With Great Power Comes Great Responsibility." -- Spider Man's Uncle
 
 This document is a tutorial about Nim's macro system.
 A macro is a function that is executed at compile-time and transforms

lib/packages/docutils/highlite.nim

@@ -125,9 +125,7 @@ proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) =
   g.length = 0
   g.state = low(TokenClass)
   g.lang = low(SourceLanguage)
-  var pos = 0                     # skip initial whitespace:
-  while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos)
-  g.pos = pos
+  g.pos = 0
 
 proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: string) =
   initGeneralTokenizer(g, cstring(buf))

lib/packages/docutils/rst.nim

@@ -8,20 +8,23 @@
 #
 
 ## ==================================
-##                rst
+##       packages/docutils/rst
 ## ==================================
 ##
 ## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ## Nim-flavored reStructuredText and Markdown
 ## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ##
-## This module implements a `reStructuredText`:idx: (RST) parser.
+## This module implements a `reStructuredText`:idx: (RST) and
+## `Markdown`:idx: parser.
 ## A large subset is implemented with some limitations_ and
 ## `Nim-specific features`_.
-## A few `extra features`_ of the `Markdown`:idx: syntax are
-## also supported.
+## Both Markdown and RST are mark-up languages whose goal is to
+## typeset texts with complex structure, formatting and references
+## using simple plaintext representation.
 ##
-## Nim can output the result to HTML [#html]_ or Latex [#latex]_.
+## This module is also embedded into Nim compiler; the compiler can output
+## the result to HTML [#html]_ or Latex [#latex]_.
 ##
 ## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and
 ##    `nim rst2html`:cmd: for ``*.rst`` files
@@ -29,11 +32,13 @@
 ## .. [#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and
 ##    `nim rst2tex`:cmd: for ``*.rst``.
 ##
-## If you are new to RST please consider reading the following:
+## If you are new to Markdown/RST please consider reading the following:
 ##
-## 1) a short `quick introduction`_
-## 2) an `RST reference`_: a comprehensive cheatsheet for RST
-## 3) a more formal 50-page `RST specification`_.
+## 1) `Markdown Basic Syntax`_
+## 2) a long specification of Markdown: `CommonMark Spec`_
+## 3) a short `quick introduction`_ to RST
+## 4) an `RST reference`_: a comprehensive cheatsheet for RST
+## 5) a more formal 50-page `RST specification`_.
 ##
 ## Features
 ## --------
@@ -120,7 +125,13 @@
 ##
 ## * emoji / smiley symbols
 ## * Markdown tables
-## * Markdown code blocks
+## * Markdown code blocks. For them the same additional arguments as for RST
+##   code blocks can be provided (e.g. `test` or `number-lines`) but with
+##   a one-line syntax like this::
+##
+##     ```nim test number-lines=10
+##     echo "ok"
+##     ```
 ## * Markdown links
 ## * Markdown headlines
 ## * Markdown block quotes
@@ -211,6 +222,8 @@
 ## See `packages/docutils/rstgen module <rstgen.html>`_ to know how to
 ## generate HTML or Latex strings to embed them into your documents.
 ##
+## .. _Markdown Basic Syntax: https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax
+## .. _CommonMark Spec: https://spec.commonmark.org/0.30
 ## .. _quick introduction: https://docutils.sourceforge.io/docs/user/rst/quickstart.html
 ## .. _RST reference: https://docutils.sourceforge.io/docs/user/rst/quickref.html
 ## .. _RST specification: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html
@@ -253,6 +266,7 @@ type
   MsgKind* = enum          ## the possible messages
     meCannotOpenFile = "cannot open '$1'",
     meExpected = "'$1' expected",
+    meMissingClosing = "$1",
     meGridTableNotImplemented = "grid table is not implemented",
     meMarkdownIllformedTable = "illformed delimiter row of a Markdown table",
     meIllformedTable = "Illformed table: $1",
@@ -323,7 +337,10 @@ const
     ":geek:": "icon_e_geek",
     ":ugeek:": "icon_e_ugeek"
   }
-  SandboxDirAllowlist = ["image", "code", "code-block"]
+  SandboxDirAllowlist = [
+    "image", "code", "code-block", "admonition", "attention", "caution",
+    "container", "contents", "danger", "default-role", "error", "figure",
+    "hint", "important", "index", "note", "role", "tip", "title", "warning"]
 
 type
   TokType = enum
@@ -1616,35 +1633,89 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string,
       inc p.idx
     else: rstMessage(p, meExpected, postfix, line, col)
 
+proc parseMarkdownCodeblockFields(p: var RstParser): PRstNode =
+  ## Parses additional (after language string) code block parameters
+  ## in a format *suggested* in the `CommonMark Spec`_ with handling of `"`.
+  if currentTok(p).kind == tkIndent:
+    result = nil
+  else:
+    result = newRstNode(rnFieldList)
+  while currentTok(p).kind != tkIndent:
+    if currentTok(p).kind == tkWhite:
+      inc p.idx
+    else:
+      let field = newRstNode(rnField)
+      var fieldName = ""
+      while currentTok(p).kind notin {tkWhite, tkIndent, tkEof} and
+            currentTok(p).symbol != "=":
+        fieldName.add currentTok(p).symbol
+        inc p.idx
+      field.add(newRstNode(rnFieldName, @[newLeaf(fieldName)]))
+      if currentTok(p).kind == tkWhite: inc p.idx
+      let fieldBody = newRstNode(rnFieldBody)
+      if currentTok(p).symbol == "=":
+        inc p.idx
+        if currentTok(p).kind == tkWhite: inc p.idx
+        var fieldValue = ""
+        if currentTok(p).symbol == "\"":
+          while true:
+            fieldValue.add currentTok(p).symbol
+            inc p.idx
+            if currentTok(p).kind == tkEof:
+              rstMessage(p, meExpected, "\"")
+            elif currentTok(p).symbol == "\"":
+              fieldValue.add "\""
+              inc p.idx
+              break
+        else:
+          while currentTok(p).kind notin {tkWhite, tkIndent, tkEof}:
+            fieldValue.add currentTok(p).symbol
+            inc p.idx
+        fieldBody.add newLeaf(fieldValue)
+      field.add(fieldBody)
+      result.add(field)
+
 proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
   result = newRstNodeA(p, rnCodeBlock)
+  let line = curLine(p)
+  let baseCol = currentTok(p).col
+  let baseSym = currentTok(p).symbol  # usually just ```
+  inc p.idx
   result.info = lineInfo(p)
   var args = newRstNode(rnDirArg)
+  var fields: PRstNode = nil
   if currentTok(p).kind == tkWord:
     args.add(newLeaf(p))
     inc p.idx
+    fields = parseMarkdownCodeblockFields(p)
   else:
     args = nil
   var n = newLeaf("")
   while true:
-    case currentTok(p).kind
-    of tkEof:
-      rstMessage(p, meExpected, "```")
+    if currentTok(p).kind == tkEof:
+      rstMessage(p, meMissingClosing,
+                 "$1 (started at line $2)" % [baseSym, $line])
       break
-    of tkPunct, tkAdornment:
-      if currentTok(p).symbol == "```":
-        inc p.idx
-        break
-      else:
-        n.text.add(currentTok(p).symbol)
-        inc p.idx
+    elif nextTok(p).kind in {tkPunct, tkAdornment} and
+         nextTok(p).symbol[0] == baseSym[0] and
+         nextTok(p).symbol.len >= baseSym.len:
+      inc p.idx, 2
+      break
+    elif currentTok(p).kind == tkIndent:
+      n.text.add "\n"
+      if currentTok(p).ival > baseCol:
+        n.text.add " ".repeat(currentTok(p).ival - baseCol)
+      elif currentTok(p).ival < baseCol:
+        rstMessage(p, mwRstStyle,
+                   "unexpected de-indentation in Markdown code block")
+      inc p.idx
     else:
       n.text.add(currentTok(p).symbol)
       inc p.idx
   var lb = newRstNode(rnLiteralBlock)
   lb.add(n)
   result.add(args)
-  result.add(PRstNode(nil))
+  result.add(fields)
   result.add(lb)
 
 proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool =
@@ -1730,6 +1801,12 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode =
     inc i
   p.idx = i
 
+proc isMarkdownCodeBlock(p: RstParser): bool =
+  result = (roSupportMarkdown in p.s.options and
+            currentTok(p).kind in {tkPunct, tkAdornment} and
+            currentTok(p).symbol[0] == '`' and  # tilde ~ is not supported
+            currentTok(p).symbol.len >= 3)
+
 proc parseInline(p: var RstParser, father: PRstNode) =
   var n: PRstNode  # to be used in `if` condition
   let saveIdx = p.idx
@@ -1755,8 +1832,7 @@ proc parseInline(p: var RstParser, father: PRstNode) =
       addAnchorRst(p, name = linkName(n), refn = refn, reset = true,
                    anchorType=manualInlineAnchor)
       father.add(n)
-    elif roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
-      inc p.idx
+    elif isMarkdownCodeBlock(p):
       father.add(parseMarkdownCodeblock(p))
     elif isInlineMarkupStart(p, "``"):
       var n = newRstNode(rnInlineLiteral)
@@ -1816,8 +1892,7 @@ proc parseInline(p: var RstParser, father: PRstNode) =
         return
     parseWordOrRef(p, father)
   of tkAdornment, tkOther, tkWhite:
-    if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
-      inc p.idx
+    if isMarkdownCodeBlock(p):
       father.add(parseMarkdownCodeblock(p))
       return
     if roSupportSmilies in p.s.options:
@@ -2194,7 +2269,7 @@ proc findPipe(p: RstParser, start: int): bool =
 proc whichSection(p: RstParser): RstNodeKind =
   if currentTok(p).kind in {tkAdornment, tkPunct}:
     # for punctuation sequences that can be both tkAdornment and tkPunct
-    if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
+    if isMarkdownCodeBlock(p):
       return rnCodeBlock
     elif currentTok(p).symbol == "::":
       return rnLiteralBlock
@@ -2633,7 +2708,9 @@ proc parseSimpleTable(p: var RstParser): PRstNode =
       # fix rnTableDataCell -> rnTableHeaderCell for previous table rows:
       for nRow in 0 ..< result.sons.len:
         for nCell in 0 ..< result.sons[nRow].len:
-          result.sons[nRow].sons[nCell].kind = rnTableHeaderCell
+          template cell: PRstNode = result.sons[nRow].sons[nCell]
+          cell = PRstNode(kind: rnTableHeaderCell, sons: cell.sons,
+                          span: cell.span, anchor: cell.anchor)
     if currentTok(p).kind == tkEof: break
     let tabRow = parseSimpleTableRow(p, cols, colChar)
     result.add tabRow
@@ -2892,11 +2969,19 @@ proc parseSection(p: var RstParser, result: PRstNode) =
       if currInd(p) == currentTok(p).ival:
         inc p.idx
       elif currentTok(p).ival > currInd(p):
-        pushInd(p, currentTok(p).ival)
-        var a = newRstNodeA(p, rnBlockQuote)
-        parseSection(p, a)
-        result.add(a)
-        popInd(p)
+        if roPreferMarkdown in p.s.options:  # Markdown => normal paragraphs
+          if currentTok(p).ival - currInd(p) >= 4:
+            rstMessage(p, mwRstStyle,
+                       "Markdown indented code not implemented")
+          pushInd(p, currentTok(p).ival)
+          parseSection(p, result)
+          popInd(p)
+        else:  # RST mode => block quotes
+          pushInd(p, currentTok(p).ival)
+          var a = newRstNodeA(p, rnBlockQuote)
+          parseSection(p, a)
+          result.add(a)
+          popInd(p)
       else:
         while currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent:
           inc p.idx  # skip blank lines

tests/stdlib/thighlite.nim

@@ -12,6 +12,12 @@ block: # Nim tokenizing
        @[("\"\"\"ok1\\nok2\\nok3\"\"\"", gtLongStringLit)
       ])
 
+  test "whitespace at beginning of line is preserved":
+    check("  discard 1".tokenize(langNim) ==
+       @[("  ", gtWhitespace), ("discard", gtKeyword), (" ", gtWhitespace),
+         ("1", gtDecNumber)
+       ])
+
 block: # Cmd (shell) tokenizing
   test "cmd with dollar and output":
     check(

tests/stdlib/trst.nim

@@ -24,8 +24,11 @@ import unittest, strutils
 import std/private/miscdollars
 import os
 
+const preferMarkdown = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled}
+const preferRst = {roSupportMarkdown, roNimFile, roSandboxDisabled}
+
 proc toAst(input: string,
-            rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled},
+            rstOptions: RstParseOptions = preferMarkdown,
             error: ref string = nil,
             warnings: ref seq[string] = nil): string =
   ## If `error` is nil then no errors should be generated.
@@ -451,7 +454,7 @@ suite "RST parsing":
         >   - y
         >
         > Paragraph.
-        """.toAst == dedent"""
+        """.toAst(rstOptions = preferRst) == dedent"""
           rnMarkdownBlockQuote
             rnMarkdownBlockQuoteItem  quotationDepth=1
               rnInner
@@ -468,6 +471,93 @@ suite "RST parsing":
                   rnLeaf  '.'
           """)
 
+  test "Markdown code blocks with more > 3 backticks":
+    check(dedent"""
+        ````
+        let a = 1
+        ```
+        ````""".toAst ==
+      dedent"""
+        rnCodeBlock
+          [nil]
+          [nil]
+          rnLiteralBlock
+            rnLeaf  '
+        let a = 1
+        ```'
+      """)
+
+  test "Markdown code blocks with Nim-specific arguments":
+    check(dedent"""
+        ```nim number-lines=1 test
+        let a = 1
+        ```""".toAst ==
+      dedent"""
+        rnCodeBlock
+          rnDirArg
+            rnLeaf  'nim'
+          rnFieldList
+            rnField
+              rnFieldName
+                rnLeaf  'number-lines'
+              rnFieldBody
+                rnLeaf  '1'
+            rnField
+              rnFieldName
+                rnLeaf  'test'
+              rnFieldBody
+          rnLiteralBlock
+            rnLeaf  '
+        let a = 1'
+        """)
+
+    check(dedent"""
+        ```nim test = "nim c $1"  number-lines = 1
+        let a = 1
+        ```""".toAst ==
+      dedent"""
+        rnCodeBlock
+          rnDirArg
+            rnLeaf  'nim'
+          rnFieldList
+            rnField
+              rnFieldName
+                rnLeaf  'test'
+              rnFieldBody
+                rnLeaf  '"nim c $1"'
+            rnField
+              rnFieldName
+                rnLeaf  'number-lines'
+              rnFieldBody
+                rnLeaf  '1'
+          rnLiteralBlock
+            rnLeaf  '
+        let a = 1'
+        """)
+
+  test "additional indentation < 4 spaces is handled fine":
+    check(dedent"""
+        Indentation
+
+          ```nim
+            let a = 1
+          ```""".toAst ==
+      dedent"""
+        rnInner
+          rnParagraph
+            rnLeaf  'Indentation'
+          rnParagraph
+            rnCodeBlock
+              rnDirArg
+                rnLeaf  'nim'
+              [nil]
+              rnLiteralBlock
+                rnLeaf  '
+          let a = 1'
+      """)
+      # | |
+      # |  \ indentation of exactly two spaces before 'let a = 1'
+
   test "option list has priority over definition list":
     check(dedent"""
         --defusages
@@ -562,7 +652,7 @@ suite "RST parsing":
 
          notAcomment1
          notAcomment2
-        someParagraph""".toAst ==
+        someParagraph""".toAst(rstOptions = preferRst) ==
       dedent"""
         rnInner
           rnBlockQuote
@@ -574,6 +664,25 @@ suite "RST parsing":
             rnLeaf  'someParagraph'
         """)
 
+  test "check that additional line right after .. ends comment (Markdown mode)":
+    # in Markdown small indentation does not matter so this should
+    # just be split to 2 paragraphs.
+    check(dedent"""
+        ..
+
+         notAcomment1
+         notAcomment2
+        someParagraph""".toAst ==
+      dedent"""
+        rnInner
+          rnInner
+            rnLeaf  'notAcomment1'
+            rnLeaf  ' '
+            rnLeaf  'notAcomment2'
+          rnParagraph
+            rnLeaf  'someParagraph'
+        """)
+
   test "but blank lines after 2nd non-empty line don't end the comment":
     check(dedent"""
         ..
@@ -592,7 +701,7 @@ suite "RST parsing":
 
         ..
 
-          someBlockQuote""".toAst ==
+          someBlockQuote""".toAst(rstOptions = preferRst) ==
       dedent"""
         rnInner
           rnAdmonition  adType=note

tests/stdlib/trstgen.nim

@@ -9,8 +9,13 @@ import ../../lib/packages/docutils/rst
 import unittest, strutils, strtabs
 import std/private/miscdollars
 
+const
+  NoSandboxOpts = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled}
+  preferMarkdown = {roPreferMarkdown, roSupportMarkdown, roNimFile}
+  preferRst = {roSupportMarkdown, roNimFile}
+
 proc toHtml(input: string,
-            rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile},
+            rstOptions: RstParseOptions = preferMarkdown,
             error: ref string = nil,
             warnings: ref seq[string] = nil): string =
   ## If `error` is nil then no errors should be generated.
@@ -47,9 +52,6 @@ proc optionListLabel(opt: string): string =
   opt &
   "</span></tt></div>"
 
-const
-  NoSandboxOpts = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled}
-
 
 suite "YAML syntax highlighting":
   test "Basics":
@@ -1180,7 +1182,7 @@ Test1
         "input(8, 4) Warning: language 'anotherLang' not supported"
         ])
     check(output == "<pre class = \"listing\">anything</pre>" &
-                    "<p><pre class = \"listing\">\nsomeCode\n</pre> </p>")
+                    "<p><pre class = \"listing\">\nsomeCode</pre> </p>")
 
   test "RST admonitions":
     # check that all admonitions are implemented
@@ -1321,7 +1323,7 @@ Test1
       That was a transition.
     """
     let output1 = input1.toHtml(
-      NoSandboxOpts
+      preferRst
     )
     doAssert "<p id=\"target000\""     in output1
     doAssert "<ul id=\"target001\""    in output1
@@ -1543,7 +1545,7 @@ Test1
             """<td>text</td></tr>""" & "\n</tbody></table>")
 
   test "Field list: body after newline":
-    let output = dedent """
+    let output = dedent"""
       :field:
         text1""".toHtml
     check "<table class=\"docinfo\"" in output

tools/kochdocs.nim

@@ -107,18 +107,18 @@ proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options
   let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input
   execFold(desc, cmd)
 
-proc getRst2html(): seq[string] =
+proc getMd2html(): seq[string] =
   for a in walkDirRecFilter("doc"):
     let path = a.path
-    if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin
-        ["docs.rst", "nimfix.rst",
-         "docstyle.rst" # docstyle.rst shouldn't be converted to html separately;
-                        # it's included in contributing.rst.
+    if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin
+        ["docs.md", "nimfix.md",
+         "docstyle.md" # docstyle.md shouldn't be converted to html separately;
+                       # it's included in contributing.md.
         ]:
           # maybe we should still show nimfix, could help reviving it
           # `docs` is redundant with `overview`, might as well remove that file?
       result.add path
-  doAssert "doc/manual/var_t_return.rst".unixToNativePath in result # sanity check
+  doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check
 
 const
   rstPdfList = """
@@ -253,13 +253,13 @@ proc buildDocPackages(nimArgs, destPath: string) =
 
 proc buildDoc(nimArgs, destPath: string) =
   # call nim for the documentation:
-  let rst2html = getRst2html()
+  let rst2html = getMd2html()
   var
     commands = newSeq[string](rst2html.len + len(doc0) + len(doc) + withoutIndex.len)
     i = 0
   let nim = findNim().quoteShell()
   for d in items(rst2html):
-    commands[i] = nim & " rst2html $# --git.url:$# -o:$# --index:on $#" %
+    commands[i] = nim & " md2html $# --git.url:$# -o:$# --index:on $#" %
       [nimArgs, gitUrl,
       destPath / changeFileExt(splitFile(d).name, "html"), d]
     i.inc