mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-08-29 18:11:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

testament: include extra options in test name (#19801)

Browse Source 
there's currently no (simple) way to disambiguate which option failed
This commit is contained in:
Jacek Sieka
2022-05-19 17:11:53 +02:00
committed by GitHub
parent 06f02bb771
commit 63cca93ea9
2 changed files with 44 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
testament/categories.nim
View File

@@ -437,7 +437,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) =
if pkg.allowFailure: if pkg.allowFailure:
inc r.passed inc r.passed
inc r.failedButAllowed inc r.failedButAllowed
addResult(r, test, targetC, "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure)
continue continue
outp outp
@@ -450,21 +450,21 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) =
discard tryCommand("nimble install --depsOnly -y", maxRetries = 3) discard tryCommand("nimble install --depsOnly -y", maxRetries = 3)
discard tryCommand(pkg.cmd, reFailed = reBuildFailed) discard tryCommand(pkg.cmd, reFailed = reBuildFailed)
inc r.passed inc r.passed
r.addResult(test, targetC, "", "", reSuccess, allowFailure = pkg.allowFailure) r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure)
errors = r.total - r.passed errors = r.total - r.passed
if errors == 0: if errors == 0:
r.addResult(packageFileTest, targetC, "", "", reSuccess) r.addResult(packageFileTest, targetC, "", "", "", reSuccess)
else: else:
r.addResult(packageFileTest, targetC, "", "", reBuildFailed) r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed)
except JsonParsingError: except JsonParsingError:
errors = 1 errors = 1
r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed) r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed)
raise raise
except ValueError: except ValueError:
errors = 1 errors = 1
r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed) r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed)
raise # bug #18805 raise # bug #18805
finally: finally:
if errors == 0: removeDir(packagesDir) if errors == 0: removeDir(packagesDir)

79
testament/testament.nim
View File

@@ -259,7 +259,8 @@ Tests skipped: $4 / $1 <br />
""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] """ % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped]
proc addResult(r: var TResults, test: TTest, target: TTarget, proc addResult(r: var TResults, test: TTest, target: TTarget,
expected, given: string, successOrig: TResultEnum, allowFailure = false, givenSpec: ptr TSpec = nil) = extraOptions, expected, given: string, successOrig: TResultEnum,
allowFailure = false, givenSpec: ptr TSpec = nil) =
# instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need
# instead of having to pass individual fields, or abusing existing ones like expected vs given. # instead of having to pass individual fields, or abusing existing ones like expected vs given.
# test.name is easier to find than test.name.extractFilename # test.name is easier to find than test.name.extractFilename
@@ -269,6 +270,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
if allowFailure: if allowFailure:
name.add " (allowed to fail) " name.add " (allowed to fail) "
if test.options.len > 0: name.add ' ' & test.options if test.options.len > 0: name.add ' ' & test.options
if extraOptions.len > 0: name.add ' ' & extraOptions
let duration = epochTime() - test.startTime let duration = epochTime() - test.startTime
let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout
@@ -333,7 +335,8 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
discard waitForExit(p) discard waitForExit(p)
close(p) close(p)
proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest,
target: TTarget, extraOptions: string) =
let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}" let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}"
var covered = initIntSet() var covered = initIntSet()
for line in splitLines(given.nimout): for line in splitLines(given.nimout):
@@ -367,10 +370,10 @@ proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest,
e.add ": " e.add ": "
e.add expected.inlineErrors[j].msg e.add expected.inlineErrors[j].msg
r.addResult(test, target, e, given.nimout, reMsgsDiffer) r.addResult(test, target, extraOptions, e, given.nimout, reMsgsDiffer)
break coverCheck break coverCheck
r.addResult(test, target, "", given.msg, reSuccess) r.addResult(test, target, extraOptions, "", given.msg, reSuccess)
inc(r.passed) inc(r.passed)
proc nimoutCheck(expected, given: TSpec): bool = proc nimoutCheck(expected, given: TSpec): bool =
@@ -381,22 +384,23 @@ proc nimoutCheck(expected, given: TSpec): bool =
elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
result = false result = false
proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
target: TTarget, extraOptions: string) =
if expected.inlineErrors.len > 0: if expected.inlineErrors.len > 0:
checkForInlineErrors(r, expected, given, test, target) checkForInlineErrors(r, expected, given, test, target, extraOptions)
elif strip(expected.msg) notin strip(given.msg): elif strip(expected.msg) notin strip(given.msg):
r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
elif not nimoutCheck(expected, given): elif not nimoutCheck(expected, given):
r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
elif extractFilename(expected.file) != extractFilename(given.file) and elif extractFilename(expected.file) != extractFilename(given.file) and
"internal error:" notin expected.msg: "internal error:" notin expected.msg:
r.addResult(test, target, expected.file, given.file, reFilesDiffer) r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
elif expected.line != given.line and expected.line != 0 or elif expected.line != given.line and expected.line != 0 or
expected.column != given.column and expected.column != 0: expected.column != given.column and expected.column != 0:
r.addResult(test, target, $expected.line & ':' & $expected.column, r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column,
$given.line & ':' & $given.column, reLinesDiffer) $given.line & ':' & $given.column, reLinesDiffer)
else: else:
r.addResult(test, target, expected.msg, given.msg, reSuccess) r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess)
inc(r.passed) inc(r.passed)
proc generatedFile(test: TTest, target: TTarget): string = proc generatedFile(test: TTest, target: TTarget): string =
@@ -434,8 +438,8 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st
given.err = reCodeNotFound given.err = reCodeNotFound
echo getCurrentExceptionMsg() echo getCurrentExceptionMsg()
proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
expected: TSpec; r: var TResults) = given: var TSpec, expected: TSpec; r: var TResults) =
var expectedmsg: string = "" var expectedmsg: string = ""
var givenmsg: string = "" var givenmsg: string = ""
if given.err == reSuccess: if given.err == reSuccess:
@@ -449,7 +453,7 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
else: else:
givenmsg = "$ " & given.cmd & '\n' & given.nimout givenmsg = "$ " & given.cmd & '\n' & given.nimout
if given.err == reSuccess: inc(r.passed) if given.err == reSuccess: inc(r.passed)
r.addResult(test, target, expectedmsg, givenmsg, given.err) r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err)
proc getTestSpecTarget(): TTarget = proc getTestSpecTarget(): TTarget =
if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": if getEnv("NIM_COMPILE_TO_CPP", "false") == "true":
@@ -457,16 +461,6 @@ proc getTestSpecTarget(): TTarget =
else: else:
result = targetC result = targetC
proc checkDisabled(r: var TResults, test: TTest): bool =
if test.spec.err in {reDisabled, reJoined}:
# targetC is a lie, but parameter is required
r.addResult(test, targetC, "", "", test.spec.err)
inc(r.skipped)
inc(r.total)
result = false
else:
result = true
var count = 0 var count = 0
proc equalModuloLastNewline(a, b: string): bool = proc equalModuloLastNewline(a, b: string): bool =
@@ -474,8 +468,13 @@ proc equalModuloLastNewline(a, b: string): bool =
result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] result = a == b or b.endsWith("\n") and a == b[0 ..< ^1]
proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
target: TTarget, nimcache: string, extraOptions = "") = target: TTarget, extraOptions: string, nimcache: string) =
test.startTime = epochTime() test.startTime = epochTime()
if test.spec.err in {reDisabled, reJoined}:
r.addResult(test, target, extraOptions, "", "", test.spec.err)
inc(r.skipped)
return
template callNimCompilerImpl(): untyped = template callNimCompilerImpl(): untyped =
# xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently
# with other branches # with other branches
@@ -483,21 +482,21 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
case expected.action case expected.action
of actionCompile: of actionCompile:
var given = callNimCompilerImpl() var given = callNimCompilerImpl()
compilerOutputTests(test, target, given, expected, r) compilerOutputTests(test, target, extraOptions, given, expected, r)
of actionRun: of actionRun:
var given = callNimCompilerImpl() var given = callNimCompilerImpl()
if given.err != reSuccess: if given.err != reSuccess:
r.addResult(test, target, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
else: else:
let isJsTarget = target == targetJS let isJsTarget = target == targetJS
var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt)
if not fileExists(exeFile): if not fileExists(exeFile):
r.addResult(test, target, expected.output, r.addResult(test, target, extraOptions, expected.output,
"executable not found: " & exeFile, reExeNotFound) "executable not found: " & exeFile, reExeNotFound)
else: else:
let nodejs = if isJsTarget: findNodeJs() else: "" let nodejs = if isJsTarget: findNodeJs() else: ""
if isJsTarget and nodejs == "": if isJsTarget and nodejs == "":
r.addResult(test, target, expected.output, "nodejs binary not in PATH", r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH",
reExeNotFound) reExeNotFound)
else: else:
var exeCmd: string var exeCmd: string
@@ -528,24 +527,24 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
else: else:
buf buf
if exitCode != expected.exitCode: if exitCode != expected.exitCode:
r.addResult(test, target, "exitcode: " & $expected.exitCode, r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode,
"exitcode: " & $exitCode & "\n\nOutput:\n" & "exitcode: " & $exitCode & "\n\nOutput:\n" &
bufB, reExitcodesDiffer) bufB, reExitcodesDiffer)
elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or
(expected.outputCheck == ocSubstr and expected.output notin bufB): (expected.outputCheck == ocSubstr and expected.output notin bufB):
given.err = reOutputsDiffer given.err = reOutputsDiffer
r.addResult(test, target, expected.output, bufB, reOutputsDiffer) r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
else: else:
compilerOutputTests(test, target, given, expected, r) compilerOutputTests(test, target, extraOptions, given, expected, r)
of actionReject: of actionReject:
let given = callNimCompilerImpl() let given = callNimCompilerImpl()
cmpMsgs(r, expected, given, test, target) cmpMsgs(r, expected, given, test, target, extraOptions)
proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = "") = proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =
for target in expected.targets: for target in expected.targets:
inc(r.total) inc(r.total)
if target notin gTargets: if target notin gTargets:
r.addResult(test, target, "", "", reDisabled) r.addResult(test, target, extraOptions, "", "", reDisabled)
inc(r.skipped) inc(r.skipped)
elif simulate: elif simulate:
inc count inc count
@@ -553,16 +552,15 @@ proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions =
else: else:
let nimcache = nimcacheDir(test.name, test.options, target) let nimcache = nimcacheDir(test.name, test.options, target)
var testClone = test var testClone = test
testSpecHelper(r, testClone, expected, target, nimcache, extraOptions) testSpecHelper(r, testClone, expected, target, extraOptions, nimcache)
proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
var expected = test.spec var expected = test.spec
if expected.parseErrors.len > 0: if expected.parseErrors.len > 0:
# targetC is a lie, but a parameter is required # targetC is a lie, but a parameter is required
r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec)
inc(r.total) inc(r.total)
return return
if not checkDisabled(r, test): return
expected.targets.incl targets expected.targets.incl targets
# still no target specified at all # still no target specified at all
@@ -572,14 +570,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
for m in test.spec.matrix: for m in test.spec.matrix:
targetHelper(r, test, expected, m) targetHelper(r, test, expected, m)
else: else:
targetHelper(r, test, expected) targetHelper(r, test, expected, "")
proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} = proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} =
if not checkDisabled(r, test): return
for target in test.spec.targets: for target in test.spec.targets:
inc(r.total) inc(r.total)
var testClone = test var testClone = test
testSpecHelper(r, testClone, test.spec, target, nimcache) testSpecHelper(r, testClone, test.spec, target, "", nimcache)
proc makeTest(test, options: string, cat: Category): TTest = proc makeTest(test, options: string, cat: Category): TTest =
result.cat = cat result.cat = cat