misc testament cleanups (#14764)

* misc testament cleanups extracted cleanups from #14530

* make sure all CI failures can be searched with `FAIL: ` including megatest failures

testament/categories.nim

@@ -587,13 +587,6 @@ proc quoted(a: string): string =
   # todo: consider moving to system.nim
   result.addQuoted(a)
 
-proc normalizeExe(file: string): string =
-  # xxx common pattern, should be exposed in std/os, even if simple (error prone)
-  when defined(posix):
-    if file.len == 0: ""
-    elif DirSep in file: file else: "./" & file
-  else: file
-
 proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
   ## returns a list of tests that have problems
   var specs: seq[TSpec] = @[]
@@ -618,12 +611,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
     return
 
   var megatest: string
-  #[
-  TODO(minor):
-  get from Nim cmd
-  put outputGotten.txt, outputGotten.txt, megatest.nim there too
-  delete upon completion, maybe
-  ]#
+  # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir`
   var outDir = nimcacheDir(testsDir / "megatest", "", targetC)
   const marker = "megatest:processing: "
 
@@ -644,15 +632,17 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
   var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "")
   if exitCode != 0:
     echo "$ " & cmdLine & "\n" & buf.string
-    quit("megatest compilation failed")
+    quit(failString & "megatest compilation failed")
 
-  (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).normalizeExe)
+  (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).dup normalizeExe)
   if exitCode != 0:
     echo buf.string
-    quit("megatest execution failed")
+    quit(failString & "megatest execution failed")
 
   norm buf.string
-  writeFile("outputGotten.txt", buf.string)
+  const outputExceptedFile = "outputExpected.txt"
+  const outputGottenFile = "outputGotten.txt"
+  writeFile(outputGottenFile, buf.string)
   var outputExpected = ""
   for i, runSpec in specs:
     outputExpected.add marker & runSpec.file & "\n"
@@ -661,14 +651,14 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
   norm outputExpected
 
   if buf.string != outputExpected:
-    writeFile("outputExpected.txt", outputExpected)
-    discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt")
-    echo "output different!"
-    # outputGotten.txt, outputExpected.txt not removed on purpose for debugging.
+    writeFile(outputExceptedFile, outputExpected)
+    discard execShellCmd("diff -uNdr $1 $2" % [outputExceptedFile, outputGottenFile])
+    echo failString & "megatest output different!"
+    # outputGottenFile, outputExceptedFile not removed on purpose for debugging.
     quit 1
   else:
-    echo "output OK"
-    removeFile("outputGotten.txt")
+    echo "megatest output OK"
+    removeFile(outputGottenFile)
     removeFile(megatestFile)
   #testSpec r, makeTest("megatest", options, cat)
 

testament/testament.nim

@@ -21,6 +21,7 @@ var backendLogging = true
 var simulate = false
 
 const
+  failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs
   testsDir = "tests" & DirSep
   resultsFile = "testresults.html"
   #jsonFile = "testresults.json" # not used
@@ -279,7 +280,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
   elif success == reJoined:
     maybeStyledEcho styleDim, fgYellow, "JOINED: ", styleBright, fgCyan, name
   else:
-    maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
+    maybeStyledEcho styleBright, fgRed, failString, fgCyan, name
     maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
     maybeStyledEcho styleBright, fgRed, "Failure: ", $success
     if success in {reBuildFailed, reNimcCrash, reInstallFailed}:
@@ -701,7 +702,7 @@ proc main() =
       cmds.add(myself & runtype & quoteShell(cat) & rest)
 
     proc progressStatus(idx: int) =
-      echo "progress[all]: i: " & $idx & " / " & $cats.len & " cat: " & cats[idx]
+      echo "progress[all]: $1/$2 starting: cat: $3" % [$idx, $cats.len, cats[idx]]
 
     if simulate:
       skips = loadSkipFrom(skipFrom)

tests/misc/tunsignedconv.nim

@@ -1,10 +1,4 @@
-discard """
-  output: '''uint
-1'''
-"""
-
 # Tests unsigned literals and implicit conversion between uints and ints
-# Passes if it compiles
 
 var h8:uint8 = 128
 var h16:uint16 = 32768
@@ -53,7 +47,7 @@ block t4176:
 proc fun(): uint = cast[uint](-1)
 const x0 = fun()
 
-echo typeof(x0)
+doAssert typeof(x0) is uint
 
 discard $x0
 
@@ -62,6 +56,6 @@ discard $x0
 const x1 = cast[uint](-1)
 discard $(x1,)
 
-# bug 13698
-let n: csize = 1
-echo n.int32
+# bug #13698
+let n: csize = 1 # xxx should that be csize_t or is that essential here?
+doAssert $n.int32 == "1"

tests/testament/tshould_not_work.nim

@@ -31,3 +31,5 @@ FAIL: tests/shouldfail/ttimeout.nim C
 Failure: reTimeout
 '''
 """
+
+# xxx `--nim:../compiler/nim`, doesn't seem correct (and should also honor `testament --nim`)