Merge remote-tracking branch 'upstream/devel' into improve-xmltree

compiler/msgs.nim

@@ -439,7 +439,7 @@ const
     "RedefinitionOfLabel", "UnknownSubstitutionX",
     "LanguageXNotSupported", "FieldXNotSupported",
     "CommentXIgnored", "NilStmt",
-    "TypelessParam", "DifferentHeaps", "WriteToForeignHeap",
+    "TypelessParam", "UseBase", "WriteToForeignHeap",
     "UnsafeCode", "EachIdentIsTuple", "ShadowIdent",
     "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit",
     "GcMem", "Destructor", "LockLevel", "ResultShadowed", "User"]

compiler/scriptconfig.nim

@@ -13,7 +13,7 @@
 import
   ast, modules, passes, passaux, condsyms,
   options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs,
-  os, times
+  os, times, osproc
 
 # we support 'cmpIgnoreStyle' natively for efficiency:
 from strutils import cmpIgnoreStyle
@@ -73,6 +73,9 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext =
   cbos getLastModificationTime:
     setResult(a, toSeconds(getLastModificationTime(getString(a, 0))))
 
+  cbos rawExec:
+    setResult(a, osproc.execCmd getString(a, 0))
+
   cbconf getEnv:
     setResult(a, os.getEnv(a.getString 0))
   cbconf existsEnv:

compiler/semexprs.nim

@@ -124,6 +124,9 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
         return newSymNode(u, n.info)
     result = newSymNode(s, n.info)
   of skVar, skLet, skResult, skForVar:
+    if s.magic == mNimvm:
+      localError(n.info, "illegal context for 'nimvm' magic")
+
     markUsed(n.info, s)
     styleCheckUse(n.info, s)
     # if a proc accesses a global variable, it is not side effect free:
@@ -1769,9 +1772,14 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode =
   #   ...
   # else:
   #   ...
-  let whenNimvm = n.sons.len == 2 and n.sons[0].kind == nkElifBranch and
-      n.sons[1].kind == nkElse and n.sons[0].sons[0].kind == nkIdent and
-      lookUp(c, n.sons[0].sons[0]).magic == mNimvm
+  var whenNimvm = false
+  if n.sons.len == 2 and n.sons[0].kind == nkElifBranch and
+      n.sons[1].kind == nkElse:
+    let exprNode = n.sons[0].sons[0]
+    if exprNode.kind == nkIdent:
+      whenNimvm = lookUp(c, exprNode).magic == mNimvm
+    elif exprNode.kind == nkSym:
+      whenNimvm = exprNode.sym.magic == mNimvm
 
   for i in countup(0, sonsLen(n) - 1):
     var it = n.sons[i]

compiler/semfold.nim

@@ -640,8 +640,6 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
       of mNaN: result = newFloatNodeT(NaN, n)
       of mInf: result = newFloatNodeT(Inf, n)
       of mNegInf: result = newFloatNodeT(NegInf, n)
-      of mNimvm:
-        localError(n.info, "illegal context for 'nimvm' magic")
       else:
         if sfFakeConst notin s.flags: result = copyTree(s.ast)
     of {skProc, skMethod}:

compiler/types.nim

@@ -411,6 +411,10 @@ const
 
 const preferToResolveSymbols = {preferName, preferModuleInfo, preferGenericArg}
 
+proc addTypeFlags(name: var string, typ: PType) {.inline.} =
+  if tfShared in typ.flags: name = "shared " & name
+  if tfNotNil in typ.flags: name.add(" not nil")
+
 proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
   var t = typ
   result = ""
@@ -418,11 +422,13 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
   if prefer in preferToResolveSymbols and t.sym != nil and
        sfAnon notin t.sym.flags:
     if t.kind == tyInt and isIntLit(t):
-      return t.sym.name.s & " literal(" & $t.n.intVal & ")"
-    if prefer == preferName or t.sym.owner.isNil:
-      return t.sym.name.s
+      result = t.sym.name.s & " literal(" & $t.n.intVal & ")"
+    elif prefer == preferName or t.sym.owner.isNil:
+      result = t.sym.name.s
     else:
-      return t.sym.owner.name.s & '.' & t.sym.name.s
+      result = t.sym.owner.name.s & '.' & t.sym.name.s
+    result.addTypeFlags(t)
+    return
   case t.kind
   of tyInt:
     if not isIntLit(t) or prefer == preferExported:
@@ -563,8 +569,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     result = typeToStr[t.kind] % typeToString(t.sons[0])
   else:
     result = typeToStr[t.kind]
-  if tfShared in t.flags: result = "shared " & result
-  if tfNotNil in t.flags: result.add(" not nil")
+  result.addTypeFlags(t)
 
 proc resultType(t: PType): PType =
   assert(t.kind == tyProc)

doc/nims.txt

@@ -0,0 +1,118 @@
+================================
+          NimScript
+================================
+
+Strictly speaking, ``NimScript`` is the subset of Nim that can be evaluated
+by Nim's builtin virtual machine (VM). This VM is used for Nim's compiletime
+function evaluation features, but also replaces Nim's existing configuration
+system.
+
+So instead of a ``myproject.nim.cfg`` configuration file, you can use
+a ``myproject.nims`` file that simply contains Nim code controlling the
+compilation process.
+
+The VM cannot deal with ``importc``, the FFI is not available, so there are not
+many stdlib modules that you can use with Nim's VM. However, the following
+modules are available:
+
+* `strutils <strutils.html>`_
+* `ospaths <ospaths.html>`_
+* `math <math.html>`_
+
+The `system <system.html>`_ module in NimScript mode additionally supports
+these operations: `nimscript <nimscript.html>`_.
+
+
+NimScript as a configuration file
+=================================
+
+What is ``x.y.key = "value"`` in the configuration file
+becomes ``switch("x.y.key", "value")``. ``--option`` is ``switch("option")``.
+The ``system`` module also exports 2 ``--`` templates for convenience:
+
+.. code-block:: nim
+  --forceBuild
+  # is the same as:
+  switch("forceBuild")
+
+
+NimScript as a build tool
+=========================
+
+The ``task`` template that the ``system`` module defines allows a NimScript
+file to be used as a build tool. The following exampled defines a
+task ``build`` that is an alias for the ``c`` command:
+
+.. code-block:: nim
+  task build, "builds an example":
+    setCommand "c"
+
+
+In fact, as a convention the following tasks should be available:
+
+=========     ===================================================
+Task          Description
+=========     ===================================================
+``build``     Build the project with the required
+              backend (``c``, ``cpp`` or ``js``).
+``tests``     Runs the tests belonging to the project.
+``bench``     Runs benchmarks belonging to the project.
+=========     ===================================================
+
+
+If the task runs an external command via ``exec`` it should afterwards call
+``setCommand "nop"`` to tell the Nim compiler that nothing else needs to be
+done:
+
+.. code-block:: nim
+
+  task tests, "test regular expressions":
+    exec "nim c -r tests"
+    setCommand "nop"
+
+
+Nimble integration
+==================
+
+A ``project.nims`` file can also be used as an alternative to
+a ``project.nimble`` file to specify the meta information (for example, author,
+description) and dependencies of a Nimble package. This means you can easily
+have platform specific dependencies:
+
+.. code-block:: nim
+
+  version = "1.0"
+  author = "The green goo."
+  description = "Lexer generation and regex implementation for Nim."
+  license = "MIT"
+
+  when defined(windows):
+    requires "oldwinapi >= 1.0"
+  else:
+    requires "gtk2 >= 1.0"
+
+
+
+
+Standalone NimScript
+====================
+
+NimScript can also be used directly as a portable replacement for Bash and
+Batch files. Use ``nim e myscript.nims`` to run ``myscript.nims``. For example,
+installation of Nimble is done with this simple script:
+
+.. code-block:: nim
+
+  mode = ScriptMode.Verbose
+
+  var id = 0
+  while dirExists("nimble" & $id):
+    inc id
+
+  exec "git clone https://github.com/nim-lang/nimble.git nimble" & $id
+
+  withDir "nimble" & $id & "/src":
+    exec "nim c nimble"
+
+  mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe
+

lib/pure/ospaths.nim

@@ -10,7 +10,7 @@
 # Included by the ``os`` module but a module in its own right for NimScript
 # support.
 
-when defined(nimscript):
+when defined(nimscript) or (defined(nimdoc) and not declared(os)):
   {.pragma: rtl.}
   {.push hint[ConvFromXtoItselfNotNeeded]:off.}
 
@@ -417,6 +417,7 @@ when not declared(getEnv) or defined(nimscript):
     else:
       when defined(nimscript):
         result = cmpic(pathA, pathB)
+      elif defined(nimdoc): discard
       else:
         result = cmpIgnoreCase(pathA, pathB)
 
@@ -490,6 +491,10 @@ when not declared(getEnv) or defined(nimscript):
           add result, path[i]
           inc(i)
 
+when defined(nimdoc) and not declared(os):
+  proc getEnv(x: string): string = discard
+  proc existsFile(x: string): bool = discard
+
 when declared(getEnv) or defined(nimscript):
   proc getHomeDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect].} =
     ## Returns the home directory of the current user.
@@ -556,5 +561,5 @@ when declared(getEnv) or defined(nimscript):
       if existsFile(x): return x
     result = ""
 
-when defined(nimscript):
+when defined(nimscript) or (defined(nimdoc) and not declared(os)):
   {.pop.} # hint[ConvFromXtoItselfNotNeeded]:off

lib/system.nim

@@ -1173,12 +1173,12 @@ const
     ## "i386", "alpha", "powerpc", "powerpc64", "powerpc64el", "sparc",
     ## "amd64", "mips", "mipsel", "arm", "arm64".
 
-  nimvm* {.magic: "Nimvm".}: bool = false
+  seqShallowFlag = low(int)
+
+let nimvm* {.magic: "Nimvm".}: bool = false
     ## may be used only in "when" expression.
     ## It is true in Nim VM context and false otherwise
 
-  seqShallowFlag = low(int)
-
 proc compileOption*(option: string): bool {.
   magic: "CompileOption", noSideEffect.}
   ## can be used to determine an on|off compile-time option. Example:

lib/system/nimscript.nim

@@ -13,23 +13,51 @@
 
 template builtin = discard
 
-proc listDirs*(dir: string): seq[string] = builtin
-proc listFiles*(dir: string): seq[string] = builtin
+proc listDirs*(dir: string): seq[string] =
+  ## Lists all the subdirectories (non-recursively) in the directory `dir`.
+  builtin
+proc listFiles*(dir: string): seq[string] =
+  ## Lists all the files (non-recursively) in the directory `dir`.
+  builtin
 
-proc removeDir(dir: string) = builtin
-proc removeFile(dir: string) = builtin
-proc moveFile(src, dest: string) = builtin
-proc copyFile(src, dest: string) = builtin
-proc createDir(dir: string) = builtin
+proc removeDir(dir: string){.
+  tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin
+proc removeFile(dir: string) {.
+  tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin
+proc moveFile(src, dest: string) {.
+  tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin
+proc copyFile(src, dest: string) {.
+  tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin
+proc createDir(dir: string) {.tags: [WriteIOEffect], raises: [OSError].} = builtin
 proc getOsError: string = builtin
 proc setCurrentDir(dir: string) = builtin
 proc getCurrentDir(): string = builtin
-proc paramStr*(i: int): string = builtin
-proc paramCount*(): int = builtin
+proc rawExec(cmd: string): int {.tags: [ExecIOEffect], raises: [OSError].} =
+  builtin
+
+proc paramStr*(i: int): string =
+  ## Retrieves the ``i``'th command line parameter.
+  builtin
+
+proc paramCount*(): int =
+  ## Retrieves the number of command line parameters.
+  builtin
+
+proc switch*(key: string, val="") =
+  ## Sets a Nim compiler command line switch, for
+  ## example ``switch("checks", "on")``.
+  builtin
+
+proc getCommand*(): string =
+  ## Gets the Nim command that the compiler has been invoked with, for example
+  ## "c", "js", "build", "help".
+  builtin
+
+proc setCommand*(cmd: string) =
+  ## Sets the Nim command that should be continued with after this Nimscript
+  ## has finished.
+  builtin
 
-proc switch*(key: string, val="") = builtin
-proc getCommand*(): string = builtin
-proc setCommand*(cmd: string) = builtin
 proc cmpIgnoreStyle(a, b: string): int = builtin
 proc cmpIgnoreCase(a, b: string): int = builtin
 
@@ -37,11 +65,23 @@ proc cmpic*(a, b: string): int = cmpIgnoreCase(a, b)
 
 proc getEnv*(key: string): string = builtin
 proc existsEnv*(key: string): bool = builtin
-proc fileExists*(filename: string): bool = builtin
-proc dirExists*(dir: string): bool = builtin
 
-proc existsFile*(filename: string): bool = fileExists(filename)
-proc existsDir*(dir: string): bool = dirExists(dir)
+proc fileExists*(filename: string): bool {.tags: [ReadIOEffect].} =
+  ## Checks if the file exists.
+  builtin
+
+proc dirExists*(dir: string): bool {.
+  tags: [ReadIOEffect].} =
+  ## Checks if the directory `dir` exists.
+  builtin
+
+proc existsFile*(filename: string): bool =
+  ## An alias for ``fileExists``.
+  fileExists(filename)
+
+proc existsDir*(dir: string): bool =
+  ## An alias for ``dirExists``.
+  dirExists(dir)
 
 proc toExe*(filename: string): string =
   ## On Windows adds ".exe" to `filename`, else returns `filename` unmodified.
@@ -60,10 +100,11 @@ template `--`*(key, val: untyped) = switch(astToStr(key), strip astToStr(val))
 template `--`*(key: untyped) = switch(astToStr(key), "")
 
 type
-  ScriptMode* {.pure.} = enum
-    Silent,
-    Verbose,
-    Whatif
+  ScriptMode* {.pure.} = enum ## Controls the behaviour of the script.
+    Silent,                   ## Be silent.
+    Verbose,                  ## Be verbose.
+    Whatif                    ## Do not run commands, instead just echo what
+                              ## would have been done.
 
 var
   mode*: ScriptMode ## Set this to influence how mkDir, rmDir, rmFile etc.
@@ -80,31 +121,44 @@ template log(msg: string, body: untyped) =
     body
 
 proc rmDir*(dir: string) {.raises: [OSError].} =
+  ## Removes the directory `dir`.
   log "rmDir: " & dir:
     removeDir dir
     checkOsError()
 
-proc rmFile*(dir: string) {.raises: [OSError].} =
-  log "rmFile: " & dir:
-    removeFile dir
+proc rmFile*(file: string) {.raises: [OSError].} =
+  ## Removes the `file`.
+  log "rmFile: " & file:
+    removeFile file
     checkOsError()
 
 proc mkDir*(dir: string) {.raises: [OSError].} =
+  ## Creates the directory `dir` including all necessary subdirectories. If
+  ## the directory already exists, no error is raised.
   log "mkDir: " & dir:
     createDir dir
     checkOsError()
 
 proc mvFile*(`from`, to: string) {.raises: [OSError].} =
+  ## Moves the file `from` to `to`.
   log "mvFile: " & `from` & ", " & to:
     moveFile `from`, to
     checkOsError()
 
 proc cpFile*(`from`, to: string) {.raises: [OSError].} =
+  ## Copies the file `from` to `to`.
   log "mvFile: " & `from` & ", " & to:
     copyFile `from`, to
     checkOsError()
 
-proc exec*(command: string, input = "", cache = "") =
+proc exec*(command: string) =
+  ## Executes an external process.
+  log "exec: " & command:
+    if rawExec(command) != 0:
+      raise newException(OSError, "FAILED: " & command)
+    checkOsError()
+
+proc exec*(command: string, input: string, cache = "") =
   ## Executes an external process.
   log "exec: " & command:
     echo staticExec(command, input, cache)
@@ -166,6 +220,11 @@ proc writeTask(name, desc: string) =
 
 template task*(name: untyped; description: string; body: untyped): untyped =
   ## Defines a task. Hidden tasks are supported via an empty description.
+  ## Example:
+  ##
+  ## .. code-block:: nim
+  ##  task build, "default build is via the C backend":
+  ##    setCommand "c"
   proc `name Task`() = body
 
   let cmd = getCommand()
@@ -177,13 +236,23 @@ template task*(name: untyped; description: string; body: untyped): untyped =
     `name Task`()
 
 var
-  packageName* = ""
-  version*, author*, description*, license*, srcdir*,
-    binDir*, backend*: string
+  packageName* = ""    ## Nimble support: Set this to the package name. It
+                       ## is usually not required to do that, nims' filename is
+                       ## the default.
+  version*: string     ## Nimble support: The package's version.
+  author*: string      ## Nimble support: The package's author.
+  description*: string ## Nimble support: The package's description.
+  license*: string     ## Nimble support: The package's license.
+  srcdir*: string      ## Nimble support: The package's source directory.
+  binDir*: string      ## Nimble support: The package's binary directory.
+  backend*: string     ## Nimble support: The package's backend.
 
   skipDirs*, skipFiles*, skipExt*, installDirs*, installFiles*,
-    installExt*, bin*: seq[string] = @[]
-  requiresData*: seq[string] = @[]
+    installExt*, bin*: seq[string] = @[] ## Nimble metadata.
+  requiresData*: seq[string] = @[] ## Exposes the list of requirements for read
+                                   ## and write accesses.
 
 proc requires*(deps: varargs[string]) =
+  ## Nimble support: Call this to set the list of requirements of your Nimble
+  ## package.
   for d in deps: requiresData.add(d)

todo.txt

@@ -1,9 +1,7 @@
 version 0.11.4
 ==============
 
-- document NimScript
 - document special cased varargs[untyped] and varargs[typed]
-
 - The remaining bugs of the lambda lifting pass that is responsible to enable
   closures and closure iterators need to be fixed.
 - ``concept`` needs to be refined, a nice name for the feature is not enough.

tools/nimweb.nim

@@ -334,7 +334,7 @@ proc parseNewsTitles(inputFilename: string): seq[TRssItem] =
   result = @[]
   if not open(input, inputFilename):
     quit("Could not read $1 for rss generation" % [inputFilename])
-  finally: input.close()
+  defer: input.close()
   while input.readLine(line):
     if line =~ reYearMonthDayTitle:
       result.add(TRssItem(year: matches[0], month: matches[1], day: matches[2],
@@ -368,7 +368,7 @@ proc generateRss(outputFilename: string, news: seq[TRssItem]) =
 
   if not open(output, outputFilename, mode = fmWrite):
     quit("Could not write to $1 for rss generation" % [outputFilename])
-  finally: output.close()
+  defer: output.close()
 
   output.write("""<?xml version="1.0" encoding="utf-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom">

web/documentation.txt

@@ -21,6 +21,9 @@ Nim's Documentation
       | The user guide lists command line arguments, special features of the
         compiler, etc.
 
+    - | `NimScript <docs/nims.html>`_
+      | NimScript is the upcoming new way to configure Nim.
+
     - | `Nim Backend Integration <docs/backends.html>`_
       | The Backend Integeration guide gives further information of how Nim can
         interact with C, C++, Objective C and JavaScript.

web/website.ini

@@ -31,9 +31,9 @@ file: ticker.txt
 [Documentation]
 doc: "endb;intern;apis;lib;manual.txt;tut1;tut2;nimc;overview;filters"
 doc: "tools;niminst;nimgrep;gc;estp;idetools;docgen;koch;backends.txt"
-doc: "nimfix.txt;nimsuggest.txt;nep1.txt"
+doc: "nimfix.txt;nimsuggest.txt;nep1.txt;nims.txt"
 pdf: "manual.txt;lib;tut1;tut2;nimc;niminst;gc"
-srcdoc2: "system.nim"
+srcdoc2: "system.nim;system/nimscript;pure/ospaths"
 srcdoc2: "core/macros;pure/marshal;core/typeinfo;core/unsigned"
 srcdoc2: "impure/re;pure/sockets;pure/typetraits"
 srcdoc2: "pure/concurrency/threadpool.nim;pure/concurrency/cpuinfo.nim"