Tester tests (#9787)

* remove --lib:lib
* added some tests for testatment
* tests are addad WIP: add colors switch to tester
* meta tester is integrated

testament/specs.nim

@@ -16,7 +16,7 @@ let isTravis* = existsEnv("TRAVIS")
 let isAppVeyor* = existsEnv("APPVEYOR")
 
 proc cmdTemplate*(): string =
-  compilerPrefix & " $target --lib:lib --hints:on -d:testing --nimblePath:tests/deps $options $file"
+  compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file"
 
 type
   TTestAction* = enum

testament/tester.nim

@@ -14,6 +14,8 @@ import
   marshal, backend, parseopt, specs, htmlgen, browsers, terminal,
   algorithm, compiler/nodejs, times, sets, md5
 
+var useColors = true
+
 const
   resultsFile = "testresults.html"
   #jsonFile = "testresults.json" # not used
@@ -32,6 +34,8 @@ Options:
   --failing                 only show failing/ignored tests
   --targets:"c c++ js objc" run tests for specified targets (default: all)
   --nim:path                use a particular nim executable (default: compiler/nim)
+  --directory:dir           Change to directory dir before reading the tests or doing anything else.
+  --colors:on|off           turn messagescoloring on|off
 """ % resultsFile
 
 type
@@ -180,11 +184,30 @@ proc initResults: TResults =
   result.skipped = 0
   result.data = ""
 
-#proc readResults(filename: string): TResults = # not used
-#  result = marshal.to[TResults](readFile(filename).string)
+import macros
+
+macro ignoreStyleEcho(args: varargs[typed]): untyped =
+  let typForegroundColor = bindSym"ForegroundColor".getType
+  let typBackgroundColor = bindSym"BackgroundColor".getType
+  let typStyle = bindSym"Style".getType
+  let typTerminalCmd = bindSym"TerminalCmd".getType
+  result = newCall(bindSym"echo")
+  for arg in children(args):
+    if arg.kind == nnkNilLit: continue
+    let typ = arg.getType
+    if typ.kind != nnkEnumTy or
+       typ != typForegroundColor and
+       typ != typBackgroundColor and
+       typ != typStyle and
+       typ != typTerminalCmd:
+      result.add(arg)
+
+template maybeStyledEcho(args: varargs[untyped]): untyped =
+  if useColors:
+    styledEcho(args)
+  else:
+    ignoreStyleEcho(args)
 
-#proc writeResults(filename: string, r: TResults) = # not used
-#  writeFile(filename, $$r)
 
 proc `$`(x: TResults): string =
   result = ("Tests passed: $1 / $3 <br />\n" &
@@ -205,17 +228,17 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
                           given = given)
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
   if success == reSuccess:
-    styledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)"
+    maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)"
   elif success == reIgnored:
-    styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
+    maybeStyledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
   else:
-    styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
-    styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
-    styledEcho styleBright, fgRed, "Failure: ", $success
-    styledEcho fgYellow, "Expected:"
-    styledEcho styleBright, expected, "\n"
-    styledEcho fgYellow, "Gotten:"
-    styledEcho styleBright, given, "\n"
+    maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
+    maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
+    maybeStyledEcho styleBright, fgRed, "Failure: ", $success
+    maybeStyledEcho fgYellow, "Expected:"
+    maybeStyledEcho styleBright, expected, "\n"
+    maybeStyledEcho fgYellow, "Gotten:"
+    maybeStyledEcho styleBright, given, "\n"
 
   if existsEnv("APPVEYOR"):
     let (outcome, msg) =
@@ -295,10 +318,15 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st
     echo getCurrentExceptionMsg()
 
 proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
-  let exp = expectedNimout.strip.replace("\C\L", "\L")
-  let giv = given.nimout.strip.replace("\C\L", "\L")
-  if exp notin giv:
-    given.err = reMsgsDiffer
+  let giv = given.nimout.strip
+  var currentPos = 0
+  # Only check that nimout contains all expected lines in that order.
+  # There may be more output in nimout. It is ignored here.
+  for line in expectedNimout.strip.splitLines:
+    currentPos = giv.find(line.strip, currentPos)
+    if currentPos < 0:
+      given.err = reMsgsDiffer
+      return
 
 proc makeDeterministic(s: string): string =
   var x = splitLines(s)
@@ -324,7 +352,6 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
 
 proc testSpec(r: var TResults, test: TTest, target = targetC) =
   let tname = test.name.addFileExt(".nim")
-  #echo "TESTING ", tname
   var expected: TSpec
   if test.action != actionRunNoSpec:
     expected = parseSpec(tname)
@@ -363,6 +390,12 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) =
       var given = callCompiler(expected.cmd, test.name, test.options,
                                target)
 
+      # echo "expected.cmd: ", expected.cmd
+      # echo "nimout: ", given.nimout
+      # echo "outp:   ", given.outp
+      # echo "msg:    ", given.msg
+      # echo "err:    ", given.err
+
       if given.err != reSuccess:
         r.addResult(test, target, "", given.msg, given.err)
         continue
@@ -437,7 +470,7 @@ proc testC(r: var TResults, test: TTest) =
   # runs C code. Doesn't support any specs, just goes by exit code.
   let tname = test.name.addFileExt(".c")
   inc(r.total)
-  styledEcho "Processing ", fgCyan, extractFilename(tname)
+  maybeStyledEcho "Processing ", fgCyan, extractFilename(tname)
   var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
   if given.err != reSuccess:
     r.addResult(test, targetC, "", given.msg, given.err)
@@ -494,6 +527,7 @@ proc main() =
 
   var targetsStr = ""
 
+
   var p = initOptParser()
   p.next()
   while p.kind == cmdLongoption:
@@ -505,7 +539,17 @@ proc main() =
       targetsStr = p.val.string
       targets = parseTargets(targetsStr)
     of "nim": compilerPrefix = p.val.string
-    else: quit Usage
+    of "directory":
+      setCurrentDir(p.val.string)
+    of "colors":
+      if p.val.string == "on":
+        useColors = true
+      elif p.val.string == "off":
+        useColors = false
+      else:
+        quit Usage
+    else:
+      quit Usage
     p.next()
   if p.kind != cmdArgument: quit Usage
   var action = p.key.string.normalize

testament/tests/shouldfail/tccodecheck.nim

@@ -0,0 +1,8 @@
+discard """
+ccodecheck: "baz"
+"""
+
+proc foo(): void {.exportc: "bar".}=
+  echo "Hello World"
+
+foo()

testament/tests/shouldfail/tcolumn.nim

@@ -0,0 +1,8 @@
+discard """
+line: 8
+column: 7
+errormsg: "undeclared identifier: 'undeclared'"
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared

testament/tests/shouldfail/terrormsg.nim

@@ -0,0 +1,8 @@
+discard """
+line: 8
+column: 6
+errormsg: "wrong error message"
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared

testament/tests/shouldfail/texitcode1.nim

@@ -0,0 +1,3 @@
+discard """
+exitcode: 1
+"""

testament/tests/shouldfail/tfile.nim

@@ -0,0 +1,6 @@
+discard """
+file: "notthisfile.nim"
+errmsg: "undeclared identifier: 'undefined'"
+"""
+
+echo undefined

testament/tests/shouldfail/tline.nim

@@ -0,0 +1,8 @@
+discard """
+line: 9
+column: 6
+errormsg: "undeclared identifier: 'undeclared'"
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared

testament/tests/shouldfail/tmaxcodesize.nim

@@ -0,0 +1,5 @@
+discard """
+maxcodesize: 1
+"""
+
+echo "Hello World"

testament/tests/shouldfail/tmsg.nim

@@ -0,0 +1,6 @@
+discard """
+msg: "Hello World"
+"""
+
+static:
+  echo "something else"

testament/tests/shouldfail/tnimout.nim

@@ -0,0 +1,7 @@
+discard """
+nimout: "Hello World!"
+action: compile
+"""
+
+static:
+  echo "something else"

testament/tests/shouldfail/toutput.nim

@@ -0,0 +1,7 @@
+discard """
+output: '''
+done
+'''
+"""
+
+echo "broken"

testament/tests/shouldfail/toutputsub.nim

@@ -0,0 +1,5 @@
+discard """
+outputsub: "something else"
+"""
+
+echo "Hello World!"

testament/tests/shouldfail/tsortoutput.nim

@@ -0,0 +1,11 @@
+discard """
+sortoutput: true
+output: '''
+2
+1
+'''
+"""
+
+# this test should ensure that the output is actually sorted
+echo "2"
+echo "1"

tests/testament/tshouldfail.nim

@@ -0,0 +1,17 @@
+discard """
+cmd: "testament/tester --directory:testament --colors:off --nim:../compiler/nim category shouldfail"
+action: compile
+nimout: '''
+FAIL: tccodecheck.nim C
+FAIL: tcolumn.nim C
+FAIL: terrormsg.nim C
+FAIL: texitcode1.nim C
+FAIL: tfile.nim C
+FAIL: tline.nim C
+FAIL: tmaxcodesize.nim C
+FAIL: tnimout.nim C
+FAIL: toutput.nim C
+FAIL: toutputsub.nim C
+FAIL: tsortoutput.nim C
+'''
+"""