From cc0364e72f14531466eec367f10f72e4bcd9168e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Tue, 27 Nov 2018 10:00:32 +0100 Subject: [PATCH] Tester tests (#9787) * remove --lib:lib * added some tests for testatment * tests are addad WIP: add colors switch to tester * meta tester is integrated --- testament/specs.nim | 2 +- testament/tester.nim | 84 ++++++++++++++++----- testament/tests/shouldfail/tccodecheck.nim | 8 ++ testament/tests/shouldfail/tcolumn.nim | 8 ++ testament/tests/shouldfail/terrormsg.nim | 8 ++ testament/tests/shouldfail/texitcode1.nim | 3 + testament/tests/shouldfail/tfile.nim | 6 ++ testament/tests/shouldfail/tline.nim | 8 ++ testament/tests/shouldfail/tmaxcodesize.nim | 5 ++ testament/tests/shouldfail/tmsg.nim | 6 ++ testament/tests/shouldfail/tnimout.nim | 7 ++ testament/tests/shouldfail/toutput.nim | 7 ++ testament/tests/shouldfail/toutputsub.nim | 5 ++ testament/tests/shouldfail/tsortoutput.nim | 11 +++ tests/testament/tshouldfail.nim | 17 +++++ 15 files changed, 164 insertions(+), 21 deletions(-) create mode 100644 testament/tests/shouldfail/tccodecheck.nim create mode 100644 testament/tests/shouldfail/tcolumn.nim create mode 100644 testament/tests/shouldfail/terrormsg.nim create mode 100644 testament/tests/shouldfail/texitcode1.nim create mode 100644 testament/tests/shouldfail/tfile.nim create mode 100644 testament/tests/shouldfail/tline.nim create mode 100644 testament/tests/shouldfail/tmaxcodesize.nim create mode 100644 testament/tests/shouldfail/tmsg.nim create mode 100644 testament/tests/shouldfail/tnimout.nim create mode 100644 testament/tests/shouldfail/toutput.nim create mode 100644 testament/tests/shouldfail/toutputsub.nim create mode 100644 testament/tests/shouldfail/tsortoutput.nim create mode 100644 tests/testament/tshouldfail.nim diff --git a/testament/specs.nim b/testament/specs.nim index 8afe9d98eb..86fc8bed44 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -16,7 +16,7 @@ let isTravis* = existsEnv("TRAVIS") let isAppVeyor* = existsEnv("APPVEYOR") proc cmdTemplate*(): string = - compilerPrefix & " $target --lib:lib --hints:on -d:testing --nimblePath:tests/deps $options $file" + compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file" type TTestAction* = enum diff --git a/testament/tester.nim b/testament/tester.nim index 2a8d0f5c81..59c0171b49 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -14,6 +14,8 @@ import marshal, backend, parseopt, specs, htmlgen, browsers, terminal, algorithm, compiler/nodejs, times, sets, md5 +var useColors = true + const resultsFile = "testresults.html" #jsonFile = "testresults.json" # not used @@ -32,6 +34,8 @@ Options: --failing only show failing/ignored tests --targets:"c c++ js objc" run tests for specified targets (default: all) --nim:path use a particular nim executable (default: compiler/nim) + --directory:dir Change to directory dir before reading the tests or doing anything else. + --colors:on|off turn messagescoloring on|off """ % resultsFile type @@ -180,11 +184,30 @@ proc initResults: TResults = result.skipped = 0 result.data = "" -#proc readResults(filename: string): TResults = # not used -# result = marshal.to[TResults](readFile(filename).string) +import macros + +macro ignoreStyleEcho(args: varargs[typed]): untyped = + let typForegroundColor = bindSym"ForegroundColor".getType + let typBackgroundColor = bindSym"BackgroundColor".getType + let typStyle = bindSym"Style".getType + let typTerminalCmd = bindSym"TerminalCmd".getType + result = newCall(bindSym"echo") + for arg in children(args): + if arg.kind == nnkNilLit: continue + let typ = arg.getType + if typ.kind != nnkEnumTy or + typ != typForegroundColor and + typ != typBackgroundColor and + typ != typStyle and + typ != typTerminalCmd: + result.add(arg) + +template maybeStyledEcho(args: varargs[untyped]): untyped = + if useColors: + styledEcho(args) + else: + ignoreStyleEcho(args) -#proc writeResults(filename: string, r: TResults) = # not used -# writeFile(filename, $$r) proc `$`(x: TResults): string = result = ("Tests passed: $1 / $3
\n" & @@ -205,17 +228,17 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) if success == reSuccess: - styledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)" + maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)" elif success == reIgnored: - styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name + maybeStyledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name else: - styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name - styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" - styledEcho styleBright, fgRed, "Failure: ", $success - styledEcho fgYellow, "Expected:" - styledEcho styleBright, expected, "\n" - styledEcho fgYellow, "Gotten:" - styledEcho styleBright, given, "\n" + maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name + maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" + maybeStyledEcho styleBright, fgRed, "Failure: ", $success + maybeStyledEcho fgYellow, "Expected:" + maybeStyledEcho styleBright, expected, "\n" + maybeStyledEcho fgYellow, "Gotten:" + maybeStyledEcho styleBright, given, "\n" if existsEnv("APPVEYOR"): let (outcome, msg) = @@ -295,10 +318,15 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st echo getCurrentExceptionMsg() proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = - let exp = expectedNimout.strip.replace("\C\L", "\L") - let giv = given.nimout.strip.replace("\C\L", "\L") - if exp notin giv: - given.err = reMsgsDiffer + let giv = given.nimout.strip + var currentPos = 0 + # Only check that nimout contains all expected lines in that order. + # There may be more output in nimout. It is ignored here. + for line in expectedNimout.strip.splitLines: + currentPos = giv.find(line.strip, currentPos) + if currentPos < 0: + given.err = reMsgsDiffer + return proc makeDeterministic(s: string): string = var x = splitLines(s) @@ -324,7 +352,6 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, proc testSpec(r: var TResults, test: TTest, target = targetC) = let tname = test.name.addFileExt(".nim") - #echo "TESTING ", tname var expected: TSpec if test.action != actionRunNoSpec: expected = parseSpec(tname) @@ -363,6 +390,12 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) = var given = callCompiler(expected.cmd, test.name, test.options, target) + # echo "expected.cmd: ", expected.cmd + # echo "nimout: ", given.nimout + # echo "outp: ", given.outp + # echo "msg: ", given.msg + # echo "err: ", given.err + if given.err != reSuccess: r.addResult(test, target, "", given.msg, given.err) continue @@ -437,7 +470,7 @@ proc testC(r: var TResults, test: TTest) = # runs C code. Doesn't support any specs, just goes by exit code. let tname = test.name.addFileExt(".c") inc(r.total) - styledEcho "Processing ", fgCyan, extractFilename(tname) + maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC) if given.err != reSuccess: r.addResult(test, targetC, "", given.msg, given.err) @@ -494,6 +527,7 @@ proc main() = var targetsStr = "" + var p = initOptParser() p.next() while p.kind == cmdLongoption: @@ -505,7 +539,17 @@ proc main() = targetsStr = p.val.string targets = parseTargets(targetsStr) of "nim": compilerPrefix = p.val.string - else: quit Usage + of "directory": + setCurrentDir(p.val.string) + of "colors": + if p.val.string == "on": + useColors = true + elif p.val.string == "off": + useColors = false + else: + quit Usage + else: + quit Usage p.next() if p.kind != cmdArgument: quit Usage var action = p.key.string.normalize diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim new file mode 100644 index 0000000000..a8d216a5b2 --- /dev/null +++ b/testament/tests/shouldfail/tccodecheck.nim @@ -0,0 +1,8 @@ +discard """ +ccodecheck: "baz" +""" + +proc foo(): void {.exportc: "bar".}= + echo "Hello World" + +foo() diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim new file mode 100644 index 0000000000..f4046d58df --- /dev/null +++ b/testament/tests/shouldfail/tcolumn.nim @@ -0,0 +1,8 @@ +discard """ +line: 8 +column: 7 +errormsg: "undeclared identifier: 'undeclared'" +""" + +# test should fail because the line directive is wrong +echo undeclared diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim new file mode 100644 index 0000000000..61c08d93d3 --- /dev/null +++ b/testament/tests/shouldfail/terrormsg.nim @@ -0,0 +1,8 @@ +discard """ +line: 8 +column: 6 +errormsg: "wrong error message" +""" + +# test should fail because the line directive is wrong +echo undeclared diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim new file mode 100644 index 0000000000..1b38b4f2e2 --- /dev/null +++ b/testament/tests/shouldfail/texitcode1.nim @@ -0,0 +1,3 @@ +discard """ +exitcode: 1 +""" diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim new file mode 100644 index 0000000000..07a526c68b --- /dev/null +++ b/testament/tests/shouldfail/tfile.nim @@ -0,0 +1,6 @@ +discard """ +file: "notthisfile.nim" +errmsg: "undeclared identifier: 'undefined'" +""" + +echo undefined diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim new file mode 100644 index 0000000000..963e44fc7f --- /dev/null +++ b/testament/tests/shouldfail/tline.nim @@ -0,0 +1,8 @@ +discard """ +line: 9 +column: 6 +errormsg: "undeclared identifier: 'undeclared'" +""" + +# test should fail because the line directive is wrong +echo undeclared diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim new file mode 100644 index 0000000000..9879e41814 --- /dev/null +++ b/testament/tests/shouldfail/tmaxcodesize.nim @@ -0,0 +1,5 @@ +discard """ +maxcodesize: 1 +""" + +echo "Hello World" diff --git a/testament/tests/shouldfail/tmsg.nim b/testament/tests/shouldfail/tmsg.nim new file mode 100644 index 0000000000..4ad17fa951 --- /dev/null +++ b/testament/tests/shouldfail/tmsg.nim @@ -0,0 +1,6 @@ +discard """ +msg: "Hello World" +""" + +static: + echo "something else" diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim new file mode 100644 index 0000000000..c0e3320531 --- /dev/null +++ b/testament/tests/shouldfail/tnimout.nim @@ -0,0 +1,7 @@ +discard """ +nimout: "Hello World!" +action: compile +""" + +static: + echo "something else" diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim new file mode 100644 index 0000000000..ac0bc7a46b --- /dev/null +++ b/testament/tests/shouldfail/toutput.nim @@ -0,0 +1,7 @@ +discard """ +output: ''' +done +''' +""" + +echo "broken" diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim new file mode 100644 index 0000000000..7cc51ee8d6 --- /dev/null +++ b/testament/tests/shouldfail/toutputsub.nim @@ -0,0 +1,5 @@ +discard """ +outputsub: "something else" +""" + +echo "Hello World!" diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim new file mode 100644 index 0000000000..4ce9ce26d8 --- /dev/null +++ b/testament/tests/shouldfail/tsortoutput.nim @@ -0,0 +1,11 @@ +discard """ +sortoutput: true +output: ''' +2 +1 +''' +""" + +# this test should ensure that the output is actually sorted +echo "2" +echo "1" diff --git a/tests/testament/tshouldfail.nim b/tests/testament/tshouldfail.nim new file mode 100644 index 0000000000..02e4bfd807 --- /dev/null +++ b/tests/testament/tshouldfail.nim @@ -0,0 +1,17 @@ +discard """ +cmd: "testament/tester --directory:testament --colors:off --nim:../compiler/nim category shouldfail" +action: compile +nimout: ''' +FAIL: tccodecheck.nim C +FAIL: tcolumn.nim C +FAIL: terrormsg.nim C +FAIL: texitcode1.nim C +FAIL: tfile.nim C +FAIL: tline.nim C +FAIL: tmaxcodesize.nim C +FAIL: tnimout.nim C +FAIL: toutput.nim C +FAIL: toutputsub.nim C +FAIL: tsortoutput.nim C +''' +"""