make testament a tool we ship with Nim; fixes #12084 (#12088)

* make testament a tool we ship with Nim; fixes #12084
* moved config to fit
* adapt testament tests to use the testament binary

changelog.md

@@ -73,6 +73,9 @@ type
 - The Nim compiler now warns about unused module imports. You can use a
   top level ``{.used.}`` pragma in the module that you want to be importable
   without producing this warning.
+- The "testament" testing tool's name was changed
+  from `tester` to `testament` and is generally available as a tool to run Nim
+  tests automatically.
 
 
 ### Compiler changes

compiler/installer.ini

@@ -87,6 +87,7 @@ Files: "bin/nimble.exe"
 Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
 Files: "bin/nimpretty.exe"
+Files: "bin/testament.exe"
 
 Files: "koch.exe"
 Files: "finish.exe"

koch.nim

@@ -176,6 +176,7 @@ proc bundleWinTools(args: string) =
   nimCompile("tools/nimgrab.nim", options = "-d:ssl " & args)
   nimCompile("tools/nimgrep.nim", options = args)
   bundleC2nim(args)
+  nimCompile("testament/testament.nim", options = args)
   when false:
     # not yet a tool worth including
     nimCompile(r"tools\downloader.nim",
@@ -217,6 +218,8 @@ proc buildTools(args: string = "") =
                  options = "-d:release " & args)
   nimCompileFold("Compile nimfind", "tools/nimfind.nim",
                  options = "-d:release " & args)
+  nimCompileFold("Compile testament", "testament/testament.nim",
+                 options = "-d:release " & args)
 
 proc nsis(latest: bool; args: string) =
   bundleNimbleExe(latest, args)
@@ -417,8 +420,8 @@ proc winRelease*() =
 template `|`(a, b): string = (if a.len > 0: a else: b)
 
 proc tests(args: string) =
-  nimexec "cc --opt:speed testament/tester"
-  let tester = quoteShell(getCurrentDir() / "testament/tester".exe)
+  nimexec "cc --opt:speed testament/testament"
+  let tester = quoteShell(getCurrentDir() / "testament/testament".exe)
   let success = tryExec tester & " " & (args|"all")
   if not success:
     quit("tests failed", QuitFailure)
@@ -482,7 +485,7 @@ proc runCI(cmd: string) =
     kochExecFold("boot -d:release -d:nimHasLibFFI", "boot -d:release -d:nimHasLibFFI")
 
   if getEnv("NIM_TEST_PACKAGES", "false") == "true":
-    execFold("Test selected Nimble packages", "nim c -r testament/tester cat nimble-packages")
+    execFold("Test selected Nimble packages", "nim c -r testament/testament cat nimble-packages")
   else:
     buildTools() # altenatively, kochExec "tools --toolsNoNimble"
 
@@ -490,10 +493,10 @@ proc runCI(cmd: string) =
     execFold("Test nimscript", "nim e tests/test_nimscript.nims")
     when defined(windows):
       # note: will be over-written below
-      execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester")
+      execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament")
 
     # main bottleneck here
-    execFold("Run tester", "nim c -r -d:nimCoroutines testament/tester --pedantic all -d:nimCoroutines")
+    execFold("Run tester", "nim c -r -d:nimCoroutines testament/testament --pedantic all -d:nimCoroutines")
 
     execFold("Run nimdoc tests", "nim c -r nimdoc/tester")
     execFold("Run nimpretty tests", "nim c -r nimpretty/tester.nim")

testament/categories.nim

@@ -717,3 +717,15 @@ proc processCategory(r: var TResults, cat: Category,
       inc testsRun
     if testsRun == 0:
       echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run"
+
+proc processPattern(r: var TResults, pattern, options: string; simulate: bool) =
+  var testsRun = 0
+  for name in walkPattern(pattern):
+    if simulate:
+      echo "Detected test: ", name
+    else:
+      var test = makeTest(name, options, Category"pattern")
+      testSpec r, test
+    inc testsRun
+  if testsRun == 0:
+    echo "no tests were found for pattern: ", pattern

testament/testament.nim

@@ -1,6 +1,6 @@
 #
 #
-#            Nim Tester
+#            Nim Testament
 #        (c) Copyright 2017 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
@@ -25,9 +25,10 @@ const
   resultsFile = "testresults.html"
   #jsonFile = "testresults.json" # not used
   Usage = """Usage:
-  tester [options] command [arguments]
+  testament [options] command [arguments]
 
 Command:
+  p|pat|pattern <glob>        run all the tests matching the given pattern
   all                         run all tests
   c|cat|category <category>   run all the tests of a certain category
   r|run <test>                run single test file
@@ -435,6 +436,10 @@ proc testSpecHelper(r: var TResults, test: TTest, expected: TSpec, target: TTarg
           if isJsTarget:
             exeCmd = nodejs
             args = concat(@[exeFile], args)
+          elif defined(posix) and not exeFile.contains('/'):
+            # "security" in Posix is actually just a euphemism
+            # for "unproductive arbitrary shit"
+            exeCmd = "./" & exeFile
           else:
             exeCmd = exeFile
           var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input)
@@ -594,7 +599,7 @@ proc main() =
 
   var p = initOptParser()
   p.next()
-  while p.kind == cmdLongoption:
+  while p.kind in {cmdLongoption, cmdShortOption}:
     case p.key.string.normalize
     of "print", "verbose": optPrintResults = true
     of "failing": optFailing = true
@@ -646,7 +651,7 @@ proc main() =
   of "all":
     #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false)
 
-    var myself = quoteShell(findExe("testament" / "tester"))
+    var myself = quoteShell(findExe("testament" / "testament"))
     if targetsStr.len > 0:
       myself &= " " & quoteShell("--targets:" & targetsStr)
 
@@ -694,6 +699,11 @@ proc main() =
     var cat = Category(p.key)
     p.next
     processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false)
+  of "p", "pat", "pattern":
+    skips = loadSkipFrom(skipFrom)
+    let pattern = p.key
+    p.next
+    processPattern(r, pattern, p.cmdLineRest.string, simulate)
   of "r", "run":
     # at least one directory is required in the path, to use as a category name
     let pathParts = split(p.key.string, {DirSep, AltSep})

tests/testament/tshouldfail.nim

@@ -1,5 +1,5 @@
 discard """
-cmd: "testament/tester --directory:testament --colors:off --backendLogging:off --nim:../compiler/nim category shouldfail"
+cmd: "testament/testament --directory:testament --colors:off --backendLogging:off --nim:../compiler/nim category shouldfail"
 action: compile
 nimout: '''
 FAIL: tests/shouldfail/tccodecheck.nim C