# # # Nim Tester # (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## Include for the tester that contains test suites that test special features ## of the compiler. # included from testament.nim import important_packages import std/[strformat, strutils, tables] from std/sequtils import filterIt const specialCategories = [ "assert", "async", "debugger", "dll", "examples", "gc", "io", "js", "ic", "ic_disabled", "lib", "manyloc", "nimble-packages", "niminaction", "threads", "untestable", # see trunner_special "testdata", "nimcache", "coroutines", "osproc", "shouldfail", "destructor" ] proc isTestFile*(file: string): bool = let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") # --------------------- DLL generation tests ---------------------------------- proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = const rpath = when defined(macosx): " --passL:-rpath --passL:@loader_path" else: "" var test1 = makeTest("lib/nimrtl.nim", options & " --outdir:tests/dll", cat) test1.spec.action = actionCompile testSpec c, test1 var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) test3.spec.action = actionCompile testSpec c, test3 var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 # windows looks in the dir of the exe (yay!): when not defined(windows): # posix relies on crappy LD_LIBRARY_PATH (ugh!): const libpathenv = when defined(haiku): "LIBRARY_PATH" else: "LD_LIBRARY_PATH" var libpath = getEnv(libpathenv) # Temporarily add the lib directory to LD_LIBRARY_PATH: putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) defer: putEnv(libpathenv, libpath) testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: # hcr tests var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) # test segfaults for now but compiles: if isOrc: basicHcrTest.spec.action = actionCompile testSpec r, basicHcrTest # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, hcri.options, nimcache, getTestSpecTarget()) hcri.testArgs = cmd.parseCmdLine testSpec r, hcri proc dllTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() runBasicDLLTest c, r, cat, options & " --mm:refc" runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" runBasicDLLTest c, r, cat, options, isOrc = true runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true when not defined(windows) and not defined(osx): # boehm library linking broken on macos 13 # still cannot find a recent Windows version of boehm.dll: runBasicDLLTest c, r, cat, options & " --gc:boehm" runBasicDLLTest c, r, cat, options & " -d:release --gc:boehm" # ------------------------------ GC tests ------------------------------------- proc gcTests(r: var TResults, cat: Category, options: string) = template run(filename, extraOptions: untyped) = testSpec r, makeTest("tests/gc" / filename, options & extraOptions, cat) # The matrix every gc test file goes through: refc (debug + realtime-release) # and orc (debug + release). This is the coverage we actually rely on today. template test(filename: untyped) = run filename, " --mm:refc" run filename, " -d:release -d:useRealtimeGC --mm:refc" run filename, " --gc:orc" run filename, " --gc:orc -d:release" # markAndSweep and boehm are legacy collectors. Exercising them for every gc # test file tripled this category's CI cost for little added signal, so only # `gctest` keeps them alive. `gctest` does not build under orc. template testLegacyGc(filename: untyped) = run filename, " --mm:refc" run filename, " -d:release -d:useRealtimeGC --mm:refc" run filename, " --gc:markAndSweep" run filename, " -d:release --gc:markAndSweep" when not defined(windows) and not defined(android) and not defined(osx): # boehm linking is broken on macOS 13 and there is no usable boehm.dll for # Windows, so those platforms skip it. run filename, " --gc:boehm" run filename, " -d:release --gc:boehm" testLegacyGc "gctest" test "foreign_thr" test "gcemscripten" test "growobjcrash" test "gcbench" test "gcleak" test "gcleak2" test "gcleak3" test "gcleak4" # Disabled because it works and takes too long to run: #test "gcleak5" test "weakrefs" test "cycleleak" test "closureleak" test "refarrayleak" test "tlists" test "thavlak" test "stackrefleak" test "cyclecollector" test "trace_globals" test "tfinalizers" # ------------------------- threading tests ----------------------------------- proc threadTests(r: var TResults, cat: Category, options: string) = template test(filename: untyped) = testSpec r, makeTest(filename, options, cat) testSpec r, makeTest(filename, options & " -d:release", cat) testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat) for t in os.walkFiles("tests/threads/t*.nim"): test(t) # ------------------------- IO tests ------------------------------------------ proc ioTests(r: var TResults, cat: Category, options: string) = # We need readall_echo to be compiled for this test to run. # dummy compile result: var c = initResults() testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) # ^- why is this not appended to r? Should this be discarded? # EDIT: this should be replaced by something like in D20210524T180826, # likewise in similar instances where `testSpec c` is used, or more generally # when a test depends on another test, as it makes tests non-independent, # creating complications for batching and megatest logic. testSpec r, makeTest("tests/system/tio", options, cat) # ------------------------- async tests --------------------------------------- proc asyncTests(r: var TResults, cat: Category, options: string) = # Run async with yrc instead of the default orc; the CI already runs long # enough that we cannot afford to test both. template test(filename: untyped) = testSpec r, makeTest(filename, options & " --mm:yrc", cat) for t in os.walkFiles("tests/async/t*.nim"): test(t) # ------------------------- debugger tests ------------------------------------ proc debuggerTests(r: var TResults, cat: Category, options: string) = if fileExists("tools/nimgrep.nim"): var t = makeTest("tools/nimgrep", options & " --debugger:on", cat) t.spec.action = actionCompile # force target to C because of MacOS 10.15 SDK headers bug # https://github.com/nim-lang/Nim/pull/15612#issuecomment-712471879 t.spec.targets = {targetC} testSpec r, t # ------------------------- JS tests ------------------------------------------ proc jsTests(r: var TResults, cat: Category, options: string) = template test(filename: untyped) = testSpec r, makeTest(filename, options, cat), {targetJS} testSpec r, makeTest(filename, options & " -d:release", cat), {targetJS} for t in os.walkFiles("tests/js/t*.nim"): test(t) for testfile in ["exception/texceptions", "exception/texcpt1", "exception/texcsub", "exception/tfinally", "exception/tfinally2", "exception/tfinally3", "collections/tactiontable", "method/tmultimjs", "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", "varres/tvartup", "int/tints", "int/tunsignedinc", "async/tjsandnativeasync"]: test "tests/" & testfile & ".nim" for testfile in ["strutils", "json", "random", "times", "logging"]: test "lib/pure/" & testfile & ".nim" # ------------------------- nim in action ----------- proc testNimInAction(r: var TResults, cat: Category, options: string) = template test(filename: untyped) = testSpec r, makeTest(filename, options, cat) template testJS(filename: untyped) = testSpec r, makeTest(filename, options, cat), {targetJS} template testCPP(filename: untyped) = testSpec r, makeTest(filename, options, cat), {targetCpp} let tests = [ "niminaction/Chapter1/various1", "niminaction/Chapter2/various2", "niminaction/Chapter2/resultaccept", "niminaction/Chapter2/resultreject", "niminaction/Chapter2/explicit_discard", "niminaction/Chapter2/no_def_eq", "niminaction/Chapter2/no_iterator", "niminaction/Chapter2/no_seq_type", "niminaction/Chapter3/ChatApp/src/server", "niminaction/Chapter3/ChatApp/src/client", "niminaction/Chapter3/various3", "niminaction/Chapter6/WikipediaStats/concurrency_regex", "niminaction/Chapter6/WikipediaStats/concurrency", "niminaction/Chapter6/WikipediaStats/naive", "niminaction/Chapter6/WikipediaStats/parallel_counts", "niminaction/Chapter6/WikipediaStats/race_condition", "niminaction/Chapter6/WikipediaStats/sequential_counts", "niminaction/Chapter6/WikipediaStats/unguarded_access", "niminaction/Chapter7/Tweeter/src/tweeter", "niminaction/Chapter7/Tweeter/src/createDatabase", "niminaction/Chapter7/Tweeter/tests/database_test", "niminaction/Chapter8/sdl/sdl_test" ] when false: # Verify that the files have not been modified. Death shall fall upon # whoever edits these hashes without dom96's permission, j/k. But please only # edit when making a conscious breaking change, also please try to make your # commit message clear and notify me so I can easily compile an errata later. # --------------------------------------------------------- # Hash-checks are disabled for Nim 1.1 and beyond # since we needed to fix the deprecated unary '<' operator. const refHashes = @[ "51afdfa84b3ca3d810809d6c4e5037ba", "30f07e4cd5eaec981f67868d4e91cfcf", "d14e7c032de36d219c9548066a97e846", "b335635562ff26ec0301bdd86356ac0c", "6c4add749fbf50860e2f523f548e6b0e", "76de5833a7cc46f96b006ce51179aeb1", "705eff79844e219b47366bd431658961", "a1e87b881c5eb161553d119be8b52f64", "2d706a6ec68d2973ec7e733e6d5dce50", "c11a013db35e798f44077bc0763cc86d", "3e32e2c5e9a24bd13375e1cd0467079c", "a5452722b2841f0c1db030cf17708955", "dc6c45eb59f8814aaaf7aabdb8962294", "69d208d281a2e7bffd3eaf4bab2309b1", "ec05666cfb60211bedc5e81d4c1caf3d", "da520038c153f4054cb8cc5faa617714", "59906c8cd819cae67476baa90a36b8c1", "9a8fe78c588d08018843b64b57409a02", "8b5d28e985c0542163927d253a3e4fc9", "783299b98179cc725f9c46b5e3b5381f", "1a2b3fba1187c68d6a9bfa66854f3318", "391ff57b38d9ea6f3eeb3fe69ab539d3" ] for i, test in tests: let filename = testsDir / test.addFileExt("nim") let testHash = getMD5(readFile(filename).string) doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed: " & $(i: i, testHash: testHash, refHash: refHashes[i]) # Run the tests. for testfile in tests: test "tests/" & testfile & ".nim" let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim" testJS jsFile let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim" testCPP cppFile # ------------------------- manyloc ------------------------------------------- proc findMainFile(dir: string): string = # finds the file belonging to ".nim.cfg"; if there is no such file # it returns the some ".nim" file if there is only one: const cfgExt = ".nim.cfg" result = "" var nimFiles = 0 for kind, file in os.walkDir(dir): if kind == pcFile: if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim" elif file.endsWith(".nim"): if result.len == 0: result = file inc nimFiles if nimFiles != 1: result.setLen(0) proc manyLoc(r: var TResults, cat: Category, options: string) = for kind, dir in os.walkDir("tests/manyloc"): if kind == pcDir: when defined(windows): if dir.endsWith"nake": continue if dir.endsWith"named_argument_bug": continue let mainfile = findMainFile(dir) if mainfile != "": var test = makeTest(mainfile, options, cat) test.spec.action = actionCompile testSpec r, test proc compileExample(r: var TResults, pattern, options: string, cat: Category) = for test in os.walkFiles(pattern): var test = makeTest(test, options, cat) test.spec.action = actionCompile testSpec r, test proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = var files: seq[string] = @[] proc isValid(file: string): bool = for dir in parentDirs(file, inclusive = false): if dir.lastPathPart in ["includes", "nimcache"]: # e.g.: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim! return false let name = extractFilename(file) if name.splitFile.ext != ".nim": return false for namei in disabledFiles: # because of `LockFreeHash.nim` which has case if namei.cmpPaths(name) == 0: return false return true for testFile in os.walkDirRec(pattern): if isValid(testFile): files.add testFile files.sort # reproducible order for testFile in files: let contents = readFile(testFile) var testObj = makeTest(testFile, options, cat) #[ todo: this logic is fragile: false positives (if appears in a comment), or false negatives, e.g. `when defined(osx) and isMainModule`. Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045 for a much better way. ]# if "when isMainModule" notin contents: testObj.spec.action = actionCompile testSpec r, testObj # ----------------------------- nimble ---------------------------------------- proc listPackagesAll(): seq[NimblePackage] = result = @[] var nimbleDir = getEnv("NIMBLE_DIR") if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" let packageIndex = nimbleDir / "packages_official.json" let packageList = parseFile(packageIndex) proc findPackage(name: string): JsonNode = result = nil for a in packageList: if a["name"].str == name: return a for pkg in important_packages.packages.items: var pkg = pkg if pkg.url.len == 0: let pkg2 = findPackage(pkg.name) if pkg2 == nil: raise newException(ValueError, "Cannot find package '$#'." % pkg.name) pkg.url = pkg2["url"].str result.add pkg proc listPackages(packageFilter: string): seq[NimblePackage] = let pkgs = listPackagesAll() if packageFilter.len != 0: # xxx document `packageFilter`, seems like a bad API, # at least should be a regex; a substring match makes no sense. result = pkgs.filterIt(packageFilter in it.name) else: if testamentData0.testamentNumBatch == 0: result = pkgs else: result = @[] for i in 0.. no importer re-sem) # * an interface edit propagates to (`iface-edit`: an `*.iface.bif` # importers cookie changes and >= 2 modules' # `*.s.bif` codegen is rebuilt) # # File format (a `tests/ic/t*.nim` whose body, after the spec header, contains a # line `#? metamorphic`): # # #? metamorphic # #!FILE a.nim # proc greet*(): string = "hi" # #!FILE main.nim # `main.nim` is always the build root # import a # echo greet() # #!STEP expect: hi # #!FILE a.nim # re-emit a module to "edit" it # proc greet*(): string = "hi" # identical content # #!STEP expect: hi; noop # # `#!FILE ` blocks (re)write a module in the virtual file system; the # accumulated file set is materialised before each `#!STEP`. A `#!STEP`'s # attributes are `;`-separated, each either `key: value` or a bare flag: # expect: noop body-edit iface-edit modules: clean # The last step always also runs the clean==incremental check. type MetamorphicError = object of CatchableError resultKind: TResultEnum expected, given: string proc mmRaise(kind: TResultEnum, expected, given: string) = var e = newException(MetamorphicError, given) e.resultKind = kind e.expected = expected e.given = given raise e proc isMetamorphicIcTest(content: string): bool = result = false for line in content.splitLines: if line.strip == "#? metamorphic": return true proc snapshotDir(dir: string): Table[string, string] = ## relative path -> raw file contents, for every file under `dir`. result = initTable[string, string]() if dirExists(dir): for it in walkDirRec(dir): result[it.relativePath(dir)] = readFile(it) proc changedPaths(prev, cur: Table[string, string]): seq[string] = result = @[] for k, v in cur: if prev.getOrDefault(k) != v: result.add k for k in prev.keys: if k notin cur: result.add k proc isProvenance(path: string): bool = ## Build-provenance sidecars that legitimately differ between a fresh build and ## an edit-accumulated one (they record build history, not codegen). Excluded ## only from the cross-build clean==incremental comparison — a *no-op* edit must ## still leave even these untouched. path.endsWith(".frontend.build.nif") proc stableBinary(path: string): string = ## Contents of a linked executable past its header region, for comparing whether ## two builds produced the same *code*. Linkers embed build-time-volatile fields ## in the header (e.g. the mingw PE `TimeDateStamp` and its derived `CheckSum`), ## so two builds seconds apart differ there even with identical codegen. Skipping ## a generous fixed window keeps the clean-vs-incremental check about codegen. const headerSkip = 4096 var f: File = nil if not open(f, path, fmRead): raise newException(IOError, "cannot open: " & path) defer: close(f) if getFileSize(f) > headerSkip: setFilePos(f, headerSkip) result = readAll(f) proc changedModuleCount(changed: seq[string]): int = ## distinct modules whose codegen (`*.s.bif`) was rebuilt. var mods: seq[string] = @[] for p in changed: if p.endsWith(".s.bif"): let key = p.extractFilename.split('.')[0] if key notin mods: mods.add key result = mods.len proc runMetamorphicIcTest(r: var TResults; file: string; cat: Category; options: string) = var test = TTest(cat: cat, name: file, options: options, spec: initSpec(file), startTime: epochTime()) test.spec.targets = {targetC} inc r.total # Absolute paths: `nim ic` runs with `workingDir = buildDir`, so a relative # `--nimcache` would resolve against the build dir, not where we read it back. let buildDir = (file.changeFileExt("") & "_mm").absolutePath let nc = buildDir / "nc" let bin = buildDir / "prog".addFileExt(ExeExt) removeDir(buildDir) createDir(buildDir) template compileIc(): untyped = execCmdEx2(compilerPrefix, ["ic", "--hint:Conf:off", "--warnings:off", "--nimcache:" & nc, "--out:" & bin, "main.nim"], workingDir = buildDir) # Parse the source into a flat op list: ("file", name, content) | ("step", attrs, ""). type OpKind = enum opFile, opStep type Op = object kind: OpKind a, b: string var ops: seq[Op] = @[] block parse: var curName = "" var buf = "" template flushFile() = if curName.len > 0: ops.add Op(kind: opFile, a: curName, b: buf) curName = ""; buf = "" for raw in readFile(file).splitLines: let s = raw.strip if s.startsWith("#!FILE"): flushFile() curName = s["#!FILE".len .. ^1].strip elif s.startsWith("#!STEP"): flushFile() ops.add Op(kind: opStep, a: s["#!STEP".len .. ^1].strip) elif curName.len > 0: buf.add raw; buf.add "\n" let lastStep = block: var n = 0 for o in ops: if o.kind == opStep: inc n n var vfs = initTable[string, string]() var prevSnap = initTable[string, string]() var prevBin = "" var stepIdx = 0 try: for o in ops: if o.kind == opFile: vfs[o.a] = o.b continue inc stepIdx let where = "step " & $stepIdx # Parse step attributes. var attrs = initTable[string, string]() for part in o.a.split(';'): let p = part.strip if p.len == 0: continue let c = p.find(':') if c >= 0: attrs[p[0 ..< c].strip] = p[c+1 .. ^1].strip else: attrs[p] = "" for fn, content in vfs: writeFile(buildDir / fn, content) let (_, cout, ccode) = compileIc() if ccode != 0: mmRaise(reBuildFailed, "", where & ": `nim ic` failed:\n" & cout) let (_, rout, rcode) = execCmdEx2(bin.absolutePath, [], workingDir = buildDir) if rcode != 0: mmRaise(reBuildFailed, "", where & ": program exited with " & $rcode & ":\n" & rout) if "expect" in attrs: let want = attrs["expect"].replace("\\n", "\n") if rout.strip == want.strip: discard else: mmRaise(reOutputsDiffer, want, where & " output:\n" & rout.strip) let snap = snapshotDir(nc) let binBytes = stableBinary(bin) if stepIdx > 1: let changed = changedPaths(prevSnap, snap) if "noop" in attrs and (changed.len != 0 or binBytes != prevBin): mmRaise(reOutputsDiffer, "no artifact change", where & ": no-op edit changed " & $changed.len & " cache file(s): " & changed.join(", ")) if "body-edit" in attrs: for p in changed: if p.endsWith(".iface.bif"): mmRaise(reOutputsDiffer, "no interface change", where & ": body-only edit changed an interface cookie: " & p) if "iface-edit" in attrs: var sawIface = false for p in changed: if p.endsWith(".iface.bif"): sawIface = true if not sawIface: mmRaise(reOutputsDiffer, "interface change", where & ": interface edit changed no `*.iface.bif` cookie") if changedModuleCount(changed) < 2: mmRaise(reOutputsDiffer, "propagation to importer", where & ": interface edit did not propagate (only " & $changedModuleCount(changed) & " module rebuilt)") if "modules" in attrs: let want = parseInt(attrs["modules"]) let got = changedModuleCount(changed) if got != want: mmRaise(reOutputsDiffer, $want & " modules rebuilt", where & ": " & $got & " module(s) rebuilt") prevSnap = snap prevBin = binBytes if "clean" in attrs or stepIdx == lastStep: removeDir(nc) let (_, cout2, ccode2) = compileIc() if ccode2 != 0: mmRaise(reBuildFailed, "", where & ": clean rebuild failed:\n" & cout2) let cleanSnap = snapshotDir(nc) let cleanBin = stableBinary(bin) if cleanBin != binBytes: mmRaise(reOutputsDiffer, "clean binary == incremental binary", where & ": clean rebuild produced a different binary") var diff: seq[string] = @[] for p in changedPaths(snap, cleanSnap): if not isProvenance(p): diff.add p if diff.len != 0: mmRaise(reOutputsDiffer, "clean cache == incremental cache", where & ": clean rebuild differs in " & $diff.len & " cache file(s): " & diff.join(", ")) prevSnap = cleanSnap prevBin = cleanBin finishTest(r, test, targetC, "", "", "", reSuccess) inc r.passed except MetamorphicError: let e = (ref MetamorphicError)(getCurrentException()) finishTest(r, test, targetC, "", e.expected, e.given, e.resultKind) proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; isNavigatorTest: bool) = template editedTest() = var test = makeTest(file, options, cat) test.spec.targets = {targetC} test.spec.cmd = compilerPrefix & " ic --hint:Conf:off --warnings:off $options " & file testSpecWithNimcache(r, test, nimcache) const tempExt = "_temp.nim" for it in walkDirRec(testsDir): # `_mm` directories hold materialised modules + nimcache for metamorphic # tests; never collect their files as tests in their own right. if "_mm" in it: continue if isTestFile(it) and not it.endsWith(tempExt): let content = readFile(it) if isMetamorphicIcTest(content): runMetamorphicIcTest(r, it, cat, options) continue let nimcache = nimcacheDir(it, options, targetC) removeDir(nimcache) for fragment in content.split("#!EDIT!#"): let file = it.replace(".nim", tempExt) writeFile(file, fragment) let oldPassed = r.passed editedTest() if r.passed != oldPassed+1: break # ---------------------------------------------------------------------------- const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"] const MegaTestCat = "megatest" proc `&.?`(a, b: string): string = # candidate for the stdlib? result = if b.startsWith(a): b else: a & b proc processSingleTest(r: var TResults, cat: Category, options, test: string, targets: set[TTarget], targetsSet: bool) = var targets = targets if not targetsSet: let target = if cat.string.normalize == "js": targetJS else: targetC targets = {target} doAssert fileExists(test), test & " test does not exist" testSpec r, makeTest(test, options, cat), targets proc isJoinableSpec(spec: TSpec): bool = # xxx simplify implementation using a whitelist of fields that are allowed to be # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576. result = useMegatest and not spec.sortoutput and spec.action == actionRun and not fileExists(spec.file.changeFileExt("cfg")) and not fileExists(spec.file.changeFileExt("nims")) and not fileExists(parentDir(spec.file) / "nim.cfg") and not fileExists(parentDir(spec.file) / "config.nims") and spec.cmd.len == 0 and spec.err != reDisabled and not spec.unjoinable and spec.exitCode == 0 and spec.retries == 0 and spec.input.len == 0 and spec.nimout.len == 0 and spec.nimoutFull == false and # so that tests can have `nimoutFull: true` with `nimout.len == 0` with # the meaning that they expect empty output. spec.matrix.len == 0 and spec.outputCheck != ocSubstr and spec.ccodeCheck.len == 0 and (spec.targets == {} or spec.targets == {targetC}) if result: if spec.file.readFile.contains "when isMainModule": result = false proc quoted(a: string): string = # TODO: consider moving to system.nim result = "" result.addQuoted(a) proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: string) = ## returns a list of tests that have problems #[ xxx create a reusable megatest API after abstracting out testament specific code, refs https://github.com/timotheecour/Nim/issues/655 and https://github.com/nim-lang/gtk2/pull/28; it's useful in other contexts. ]# var specs: seq[TSpec] = @[] for kind, dir in walkDir(testsDir): assert dir.startsWith(testsDir) let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin specialCategories: for file in walkDirRec(testsDir / cat): if isTestFile(file): var spec: TSpec = default(TSpec) try: spec = parseSpec(file) except ValueError: # e.g. for `tests/navigator/tincludefile.nim` which have multiple # specs; this will be handled elsewhere echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file continue if isJoinableSpec(spec): specs.add spec proc cmp(a: TSpec, b: TSpec): auto = cmp(a.file, b.file) sort(specs, cmp = cmp) # reproducible order echo "joinable specs: ", specs.len if simulate: var s = "runJoinedTest: " for a in specs: s.add a.file & " " echo s return var megatest: string = "" # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir` var outDir = nimcacheDir(testsDir / "megatest", "", targetC) template toMarker(file, i): string = "megatest:processing: [$1] $2" % [$i, file] for i, runSpec in specs: let file = runSpec.file let file2 = outDir / ("megatest_a_$1.nim" % $i) # `include` didn't work with `trecmod2.nim`, so using `import` let code = "echo $1\nstatic: echo \"CT:\", $1\n" % [toMarker(file, i).quoted] createDir(file2.parentDir) writeFile(file2, code) megatest.add "import $1\nimport $2 as megatest_b_$3\n" % [file2.quoted, file.quoted, $i] let megatestFile = testsDir / "megatest.nim" # so it uses testsDir / "config.nims" writeFile(megatestFile, megatest) let root = getCurrentDir() var args = @["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd", "--path:" & root] args.add options.parseCmdLine args.add megatestFile var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "") if exitCode != 0: echo "$ " & cmdLine & "\n" & buf quit(failString & "megatest compilation failed") (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).dup normalizeExe) if exitCode != 0: echo buf quit(failString & "megatest execution failed") const outputExceptedFile = "outputExpected.txt" const outputGottenFile = "outputGotten.txt" writeFile(outputGottenFile, buf) var outputExpected = "" for i, runSpec in specs: outputExpected.add toMarker(runSpec.file, i) & "\n" if runSpec.output.len > 0: outputExpected.add runSpec.output if not runSpec.output.endsWith "\n": outputExpected.add '\n' if buf != outputExpected: writeFile(outputExceptedFile, outputExpected) echo diffFiles(outputGottenFile, outputExceptedFile).output echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile] # outputGottenFile, outputExceptedFile not removed on purpose for debugging. quit 1 else: echo "megatest output OK" # --------------------------------------------------------------------------- proc processCategory(r: var TResults, cat: Category, options, testsDir: string, runJoinableTests: bool) = let cat2 = cat.string.normalize var handled = false if isNimRepoTests(): handled = true case cat2 of "js": # only run the JS tests on Windows or Linux because Travis is bad # and other OSes like Haiku might lack nodejs: if not defined(linux) and isTravis: discard else: jsTests(r, cat, options) of "dll": dllTests(r, cat, options & " -d:nimDebugDlOpen") of "gc": gcTests(r, cat, options) of "debugger": debuggerTests(r, cat, options) of "manyloc": manyLoc r, cat, options of "threads": threadTests r, cat, options & " --threads:on" of "io": ioTests r, cat, options of "async": asyncTests r, cat, options of "lib": testStdlib(r, "lib/pure/", options, cat) testStdlib(r, "lib/packages/docutils/", options, cat) of "examples": compileExample(r, "examples/*.nim", options, cat) compileExample(r, "examples/gtk/*.nim", options, cat) compileExample(r, "examples/talk/*.nim", options, cat) of "nimble-packages": testNimblePackages(r, cat, options) of "niminaction": testNimInAction(r, cat, options) of "ic": icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false) of "navigator": icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true) of "untestable": # These require special treatment e.g. because they depend on a third party # dependency; see `trunner_special` which runs some of those. discard else: handled = false if not handled: case cat2 of "megatest": runJoinedTest(r, cat, testsDir, options) if isNimRepoTests(): runJoinedTest(r, cat, testsDir, options & " --mm:refc") else: var testsRun = 0 var files: seq[string] = @[] for file in walkDirRec(testsDir &.? cat.string): if isTestFile(file): files.add file files.sort # give reproducible order for i, name in files: var test = makeTest(name, options, cat) if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: discard "run the test" else: test.spec.err = reJoined testSpec r, test inc testsRun if testsRun == 0: const whiteListedDirs = ["deps", "htmldocs", "pkgs"] # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users # that try an old version of choosenim. doAssert cat.string in whiteListedDirs, "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs] proc processPattern(r: var TResults, pattern, options: string; simulate: bool) = var testsRun = 0 if dirExists(pattern): for k, name in walkDir(pattern): if k in {pcFile, pcLinkToFile} and name.endsWith(".nim"): if simulate: echo "Detected test: ", name else: var test = makeTest(name, options, Category"pattern") testSpec r, test inc testsRun else: for name in walkPattern(pattern): if simulate: echo "Detected test: ", name else: var test = makeTest(name, options, Category"pattern") testSpec r, test inc testsRun if testsRun == 0: echo "no tests were found for pattern: ", pattern