diff --git a/testament/categories.nim b/testament/categories.nim index a2a0c5d5ba..e0f57df8dc 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -551,7 +551,8 @@ proc testNimblePackages(r: var TResults, cat: Category) = # ---------------------------------------------------------------------------- -const AdditionalCategories = ["debugger", "examples", "lib", "megatest"] +const AdditionalCategories = ["debugger", "examples", "lib"] +const MegaTestCat = "megatest" proc `&.?`(a, b: string): string = # candidate for the stdlib? @@ -679,22 +680,9 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = # --------------------------------------------------------------------------- -proc loadSkipFrom(name: string): seq[string] = - # One skip per line, comments start with # - # used by `nlvm` (at least) - try: - for line in lines(name): - let sline = line.strip() - if sline.len > 0 and not sline.startsWith("#"): - result.add sline - except: - echo "Could not load " & name & ", ignoring" - proc processCategory(r: var TResults, cat: Category, - options, testsDir, skipFrom: string, + options, testsDir: string, runJoinableTests: bool) = - let skips = loadSkipFrom(skipFrom) - case cat.string.normalize of "rodfiles": when false: @@ -750,9 +738,6 @@ proc processCategory(r: var TResults, cat: Category, for i, name in files: var test = makeTest(name, options, cat) - if skips.anyIt(it in name): - test.spec.err = reDisabled - if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: discard "run the test" else: diff --git a/testament/specs.nim b/testament/specs.nim index 96b02746bf..11286ceaba 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -7,13 +7,15 @@ # distribution, for details about the copyright. # -import parseutils, strutils, os, osproc, streams, parsecfg +import sequtils, parseutils, strutils, os, osproc, streams, parsecfg var compilerPrefix* = findExe("nim") let isTravis* = existsEnv("TRAVIS") let isAppVeyor* = existsEnv("APPVEYOR") +var skips*: seq[string] + type TTestAction* = enum actionRun = "run" @@ -245,3 +247,6 @@ proc parseSpec*(filename: string): TSpec = of cfgEof: break close(p) + + if skips.anyIt(it in result.file): + result.err = reDisabled diff --git a/testament/tester.nim b/testament/tester.nim index b73190eba1..f172a670d7 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -44,6 +44,7 @@ Options: --directory:dir Change to directory dir before reading the tests or doing anything else. --colors:on|off Turn messagescoloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. + --megatest:on|off Enable or disable megatest. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored """ % resultsFile @@ -380,6 +381,15 @@ proc getTestSpecTarget(): TTarget = else: return targetC +proc checkDisabled(r: var TResults, test: TTest): bool = + if test.spec.err in {reDisabled, reJoined}: + # targetC is a lie, but parameter is required + r.addResult(test, targetC, "", "", test.spec.err) + inc(r.skipped) + inc(r.total) + return + true + proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = var expected = test.spec if expected.parseErrors.len > 0: @@ -387,12 +397,8 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) inc(r.total) return - if expected.err in {reDisabled, reJoined}: - # targetC is a lie, but parameter is required - r.addResult(test, targetC, "", "", expected.err) - inc(r.skipped) - inc(r.total) - return + if not checkDisabled(r, test): return + expected.targets.incl targets # still no target specified at all if expected.targets == {}: @@ -472,6 +478,8 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = proc testC(r: var TResults, test: TTest, action: TTestAction) = # runs C code. Doesn't support any specs, just goes by exit code. + if not checkDisabled(r, test): return + let tname = test.name.addFileExt(".c") inc(r.total) maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) @@ -486,6 +494,8 @@ proc testC(r: var TResults, test: TTest, action: TTestAction) = proc testExec(r: var TResults, test: TTest) = # runs executable or script, just goes by exit code + if not checkDisabled(r, test): return + inc(r.total) let (outp, errC) = execCmdEx(test.options.strip()) var given: TSpec @@ -534,6 +544,19 @@ else: include categories +proc loadSkipFrom(name: string): seq[string] = + if name.len() == 0: return + + # One skip per line, comments start with # + # used by `nlvm` (at least) + try: + for line in lines(name): + let sline = line.strip() + if sline.len > 0 and not sline.startsWith("#"): + result.add sline + except: + echo "Could not load " & name & ", ignoring" + proc main() = os.putenv "NIMTEST_COLOR", "never" os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" @@ -544,6 +567,7 @@ proc main() = var targetsStr = "" var isMainProcess = true var skipFrom = "" + var useMegatest = true var p = initOptParser() p.next() @@ -569,6 +593,14 @@ proc main() = quit Usage of "simulate": simulate = true + of "megatest": + case p.val.string: + of "on": + useMegatest = true + of "off": + useMegatest = false + else: + quit Usage of "backendlogging": case p.val.string: of "on": @@ -608,32 +640,37 @@ proc main() = if kind == pcDir and cat notin ["testdata", "nimcache"]: cats.add cat cats.add AdditionalCategories + if useMegatest: cats.add MegaTestCat var cmds: seq[string] for cat in cats: - cmds.add(myself & " pcat " & quoteShell(cat) & rest) + let runtype = if useMegatest: " pcat " else: " cat " + cmds.add(myself & runtype & quoteShell(cat) & rest) proc progressStatus(idx: int) = echo "progress[all]: i: " & $idx & " / " & $cats.len & " cat: " & cats[idx] if simulate: + skips = loadSkipFrom(skipFrom) for i, cati in cats: progressStatus(i) - processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = false) + processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false) else: quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus) of "c", "cat", "category": + skips = loadSkipFrom(skipFrom) var cat = Category(p.key) p.next - processCategory(r, cat, p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = true) + processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true) of "pcat": + skips = loadSkipFrom(skipFrom) # 'pcat' is used for running a category in parallel. Currently the only # difference is that we don't want to run joinable tests here as they # are covered by the 'megatest' category. isMainProcess = false var cat = Category(p.key) p.next - processCategory(r, cat, p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = false) + processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false) of "r", "run": # at least one directory is required in the path, to use as a category name let pathParts = split(p.key.string, {DirSep, AltSep})