lots of testament bug fixes and improvements: (#10044)

testament/categories.nim

@@ -402,14 +402,42 @@ proc compileExample(r: var TResults, pattern, options: string, cat: Category) =
     testSpec r, test
 
 proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
-  for testFile in os.walkFiles(pattern):
-    let name = extractFilename(testFile)
-    if name notin disabledFiles:
-      let contents = readFile(testFile).string
-      var testObj = makeTest(testFile, options, cat)
-      if "when isMainModule" notin contents:
-        testObj.spec.action = actionCompile
-      testSpec r, testObj
+  var files: seq[string]
+
+  proc isValid(file: string): bool =
+    for dir in parentDirs(file, inclusive = false):
+      if dir.lastPathPart in ["includes", "nimcache"]:
+        # eg: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim!
+        return false
+    let name = extractFilename(file)
+    if name.splitFile.ext != ".nim": return false
+    for namei in disabledFiles:
+      # because of `LockFreeHash.nim` which has case
+      if namei.cmpPaths(name) == 0: return false
+    return true
+
+  for testFile in os.walkDirRec(pattern):
+    if isValid(testFile):
+      files.add testFile
+
+  files.sort # reproducible order
+  for testFile in files:
+    let contents = readFile(testFile).string
+    var testObj = makeTest(testFile, options, cat)
+    #[
+    TODO:
+    this logic is fragile:
+    false positives (if appears in a comment), or false negatives, eg
+    `when defined(osx) and isMainModule`.
+    Instead of fixing this, a much better way, is to extend
+    https://github.com/nim-lang/Nim/issues/9581 to stdlib modules as follows:
+    * add these to megatest
+    * patch compiler so `isMainModule` is true when -d:isMainModuleIsAlwaysTrue
+    That'll give speedup benefit, and we don't have to patch stdlib files.
+    ]#
+    if "when isMainModule" notin contents:
+      testObj.spec.action = actionCompile
+    testSpec r, testObj
 
 # ----------------------------- nimble ----------------------------------------
 type
@@ -517,7 +545,9 @@ proc isJoinableSpec(spec: TSpec): bool =
   result = not spec.sortoutput and
     spec.action == actionRun and
     not fileExists(spec.file.changeFileExt("cfg")) and
+    not fileExists(spec.file.changeFileExt("nims")) and
     not fileExists(parentDir(spec.file) / "nim.cfg") and
+    not fileExists(parentDir(spec.file) / "config.nims") and
     spec.cmd.len == 0 and
     spec.err != reDisabled and
     not spec.unjoinable and
@@ -535,21 +565,31 @@ proc norm(s: var string) =
     s = tmp
   s = s.strip
 
+proc isTestFile*(file: string): bool =
+  let (dir, name, ext) = splitFile(file)
+  result = ext == ".nim" and name.startsWith("t")
+
 proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
   ## returs a list of tests that have problems
   var specs: seq[TSpec] = @[]
-
   for kind, dir in walkDir(testsDir):
     assert testsDir.startsWith(testsDir)
     let cat = dir[testsDir.len .. ^1]
     if kind == pcDir and cat notin specialCategories:
-      for file in os.walkFiles(testsDir / cat / "t*.nim"):
+      for file in walkDirRec(testsDir / cat):
+        if not isTestFile(file): continue
         let spec = parseSpec(file)
         if isJoinableSpec(spec):
           specs.add spec
 
   echo "joinable specs: ", specs.len
 
+  if simulate:
+    var s = "runJoinedTest: "
+    for a in specs: s.add a.file & " "
+    echo s
+    return
+
   var megatest: string
   for runSpec in specs:
     megatest.add "import r\""
@@ -623,8 +663,8 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
   of "async":
     asyncTests r, cat, options
   of "lib":
-    testStdlib(r, "lib/pure/*.nim", options, cat)
-    testStdlib(r, "lib/packages/docutils/highlite", options, cat)
+    testStdlib(r, "lib/pure/", options, cat)
+    testStdlib(r, "lib/packages/docutils/", options, cat)
   of "examples":
     compileExample(r, "examples/*.nim", options, cat)
     compileExample(r, "examples/gtk/*.nim", options, cat)
@@ -644,7 +684,13 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
     runJoinedTest(r, cat, testsDir)
   else:
     var testsRun = 0
-    for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"):
+
+    var files: seq[string]
+    for file in walkDirRec("tests" & DirSep &.? cat.string):
+        if isTestFile(file): files.add file
+    files.sort # give reproducible order
+
+    for i, name in files:
       var test = makeTest(name, options, cat)
       if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
         discard "run the test"

testament/tester.nim

@@ -16,6 +16,7 @@ import
 
 var useColors = true
 var backendLogging = true
+var simulate = false
 
 const
   resultsFile = "testresults.html"
@@ -33,6 +34,7 @@ Arguments:
   arguments are passed to the compiler
 Options:
   --print                   also print results to the console
+  --simulate                see what tests would be run but don't run them (for debugging)
   --failing                 only show failing/ignored tests
   --targets:"c c++ js objc" run tests for specified targets (default: all)
   --nim:path                use a particular nim executable (default: compiler/nim)
@@ -219,7 +221,11 @@ proc `$`(x: TResults): string =
 
 proc addResult(r: var TResults, test: TTest, target: TTarget,
                expected, given: string, success: TResultEnum) =
-  let name = test.name.extractFilename & " " & $target & test.options
+  # test.name is easier to find than test.name.extractFilename
+  # A bit hacky but simple and works with tests/testament/tshouldfail.nim
+  var name = test.name.replace(DirSep, '/')
+  name.add " " & $target & test.options
+
   let duration = epochTime() - test.startTime
   let durationStr = duration.formatFloat(ffDecimal, precision = 8).align(11)
   if backendLogging:
@@ -376,6 +382,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
       r.addResult(test, target, "", "", reDisabled)
       inc(r.skipped)
       continue
+
+    if simulate:
+      var count {.global.} = 0
+      count.inc
+      echo "testSpec count: ", count, " expected: ", expected
+      continue
+
     case expected.action
     of actionCompile:
       var given = callCompiler(expected.getCmd, test.name, test.options, target,
@@ -477,14 +490,30 @@ proc makeTest(test, options: string, cat: Category): TTest =
   result.spec = parseSpec(addFileExt(test, ".nim"))
   result.startTime = epochTime()
 
+# TODO: fix these files
+const disabledFilesDefault = @[
+  "LockFreeHash.nim",
+  "sharedstrings.nim",
+  "tableimpl.nim",
+
+  # Error: undeclared identifier: 'hasThreadSupport'
+  "ioselectors_epoll.nim",
+  "ioselectors_kqueue.nim",
+  "ioselectors_poll.nim",
+
+  # Error: undeclared identifier: 'Timeval'
+  "ioselectors_select.nim",
+]
+
 when defined(windows):
   const
     # array of modules disabled from compilation test of stdlib.
-    disabledFiles = ["coro.nim"]
+    disabledFiles = disabledFilesDefault & @["coro.nim"]
 else:
   const
     # array of modules disabled from compilation test of stdlib.
-    disabledFiles = ["-"]
+    # TODO: why the ["-"]? (previous code should've prob used seq[string] = @[] instead)
+    disabledFiles = disabledFilesDefault & @["-"]
 
 include categories
 
@@ -521,6 +550,8 @@ proc main() =
         useColors = false
       else:
         quit Usage
+    of "simulate":
+      simulate = true
     of "backendlogging":
       case p.val.string:
       of "on":
@@ -547,16 +578,28 @@ proc main() =
 
     myself &= " " & quoteShell("--nim:" & compilerPrefix)
 
-    var cmds: seq[string] = @[]
+    var cats: seq[string]
     let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: ""
     for kind, dir in walkDir(testsDir):
       assert testsDir.startsWith(testsDir)
       let cat = dir[testsDir.len .. ^1]
       if kind == pcDir and cat notin ["testdata", "nimcache"]:
-        cmds.add(myself & " pcat " & quoteShell(cat) & rest)
-    for cat in AdditionalCategories:
+        cats.add cat
+    cats.add AdditionalCategories
+
+    var cmds: seq[string]
+    for cat in cats:
       cmds.add(myself & " pcat " & quoteShell(cat) & rest)
-    quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams})
+
+    proc progressStatus(idx: int) =
+      echo "progress[all]: i: " & $idx & " / " & $cats.len & " cat: " & cats[idx]
+
+    if simulate:
+      for i, cati in cats:
+        progressStatus(i)
+        processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false)
+    else:
+      quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
   of "c", "cat", "category":
     var cat = Category(p.key)
     p.next

tests/testament/tshouldfail.nim

@@ -2,30 +2,30 @@ discard """
 cmd: "testament/tester --directory:testament --colors:off --backendLogging:off --nim:../compiler/nim category shouldfail"
 action: compile
 nimout: '''
-FAIL: tccodecheck.nim C
+FAIL: tests/shouldfail/tccodecheck.nim C
 Failure: reCodegenFailure
 Expected:
 baz
-FAIL: tcolumn.nim C
+FAIL: tests/shouldfail/tcolumn.nim C
 Failure: reLinesDiffer
-FAIL: terrormsg.nim C
+FAIL: tests/shouldfail/terrormsg.nim C
 Failure: reMsgsDiffer
-FAIL: texitcode1.nim C
+FAIL: tests/shouldfail/texitcode1.nim C
 Failure: reExitcodesDiffer
-FAIL: tfile.nim C
+FAIL: tests/shouldfail/tfile.nim C
 Failure: reFilesDiffer
-FAIL: tline.nim C
+FAIL: tests/shouldfail/tline.nim C
 Failure: reLinesDiffer
-FAIL: tmaxcodesize.nim C
+FAIL: tests/shouldfail/tmaxcodesize.nim C
 Failure: reCodegenFailure
 max allowed size: 1
-FAIL: tnimout.nim C
+FAIL: tests/shouldfail/tnimout.nim C
 Failure: reMsgsDiffer
-FAIL: toutput.nim C
+FAIL: tests/shouldfail/toutput.nim C
 Failure: reOutputsDiffer
-FAIL: toutputsub.nim C
+FAIL: tests/shouldfail/toutputsub.nim C
 Failure: reOutputsDiffer
-FAIL: tsortoutput.nim C
+FAIL: tests/shouldfail/tsortoutput.nim C
 Failure: reOutputsDiffer
 '''
 """