use targets from test spec when running testament tests (#6687)

tests/cpp/tasync_cpp.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
   output: "hello"
 """
 

tests/cpp/tcovariancerules.nim

@@ -1,5 +1,5 @@
 discard """
-cmd: "nim cpp $file"
+targets: "cpp"
 output: '''
 cat
 cat

tests/cpp/tcppraise.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
   output: '''foo
 bar
 Need odd and >= 3 digits##

tests/cpp/tdont_init_instantiation.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
   output: ''''''
   disabled: true
 """

tests/cpp/tembarrassing_generic_failure.nim

@@ -1,4 +1,5 @@
 discard """
+  targets: "cpp"
   cmd: "nim cpp --threads:on $file"
 """
 

tests/cpp/temitlist.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
   output: '''6.0'''
 """
 

tests/cpp/tempty_generic_obj.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
   output: '''int
 float'''
 """

tests/cpp/tgen_prototype_for_importc.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
   output: '''Hello world'''
 """
 

tests/cpp/tget_subsystem.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
 """
 
 {.emit: """

tests/cpp/tnativesockets.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
 """
 
 import nativesockets

tests/cpp/treturn_array.nim

@@ -1,3 +1,6 @@
+discard """
+  targets: "cpp"
+"""
 
 # bug #2259
 type Mat4f* = array[0..15, float]

tests/cpp/tsigbreak.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
 """
 
 import tables, lists

tests/cpp/tstaticvar_via_typedesc.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
   output: "42"
 """
 

tests/cpp/ttemplatetype.nim

@@ -1,3 +1,7 @@
+discard """
+  targets: "cpp"
+"""
+
 type
   Map {.importcpp: "std::map", header: "<map>".} [T,U] = object
 

tests/cpp/tthread_createthread.nim

@@ -1,4 +1,5 @@
 discard """
+  targets: "cpp"
   cmd: "nim cpp --hints:on --threads:on $options $file"
 """
 

tests/cpp/ttypeinfo.nim

@@ -1,6 +1,6 @@
 discard """
+  targets: "cpp"
   output: '''100'''
-  cmd: "nim cpp $file"
 """
 
 import typeinfo

tests/cpp/ttypeinfo2.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
 """
 # bug #2841
 import typeinfo

tests/cpp/tvector_iterator.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim cpp $file"
+  targets: "cpp"
 """
 
 {.emit: """

tests/cpp/tvectorseq.nim

@@ -1,7 +1,7 @@
 discard """
+  targets: "cpp"
   output: '''(x: 1.0)
 (x: 0.0)'''
-  cmd: "nim cpp $file"
   disabled: "true"
 """
 

tests/testament/categories.nim

@@ -219,9 +219,9 @@ proc debuggerTests(r: var TResults, cat: Category, options: string) =
 proc jsTests(r: var TResults, cat: Category, options: string) =
   template test(filename: untyped) =
     testSpec r, makeTest(filename, options & " -d:nodejs", cat,
-                         actionRun, targetJS)
+                         actionRun), targetJS
     testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat,
-                         actionRun, targetJS)
+                         actionRun), targetJS
 
   for t in os.walkFiles("tests/js/t*.nim"):
     test(t)
@@ -245,10 +245,10 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
     testSpec r, makeTest(filename, options, cat, action)
 
   template testJS(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, actionCompile, targetJS)
+    testSpec r, makeTest(filename, options, cat, actionCompile), targetJS
 
   template testCPP(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, actionCompile, targetCPP)
+    testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP
 
   let tests = [
     "niminaction/Chapter3/ChatApp/src/server",
@@ -387,7 +387,7 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
         installStatus = waitForExitEx(installProcess)
       installProcess.close
       if installStatus != QuitSuccess:
-        r.addResult(test, "", "", reInstallFailed)
+        r.addResult(test, targetC, "", "", reInstallFailed)
         continue
 
       let
@@ -396,12 +396,12 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
         buildStatus = waitForExitEx(buildProcess)
       buildProcess.close
       if buildStatus != QuitSuccess:
-        r.addResult(test, "", "", reBuildFailed)
-      r.addResult(test, "", "", reSuccess)
-    r.addResult(packageFileTest, "", "", reSuccess)
+        r.addResult(test, targetC, "", "", reBuildFailed)
+      r.addResult(test, targetC, "", "", reSuccess)
+    r.addResult(packageFileTest, targetC, "", "", reSuccess)
   except JsonParsingError:
     echo("[Warning] - Cannot run nimble tests: Invalid package file.")
-    r.addResult(packageFileTest, "", "", reBuildFailed)
+    r.addResult(packageFileTest, targetC, "", "", reBuildFailed)
 
 
 # ----------------------------------------------------------------------------
@@ -420,7 +420,7 @@ proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
   let test = "tests" & DirSep &.? cat.string / test
   let target = if cat.string.normalize == "js": targetJS else: targetC
 
-  if existsFile(test): testSpec r, makeTest(test, options, cat, target = target)
+  if existsFile(test): testSpec r, makeTest(test, options, cat), target
   else: echo "[Warning] - ", test, " test does not exist"
 
 proc processCategory(r: var TResults, cat: Category, options: string) =

tests/testament/tester.nim

@@ -44,7 +44,6 @@ type
     name: string
     cat: Category
     options: string
-    target: TTarget
     action: TTestAction
     startTime: float
 
@@ -155,13 +154,13 @@ proc `$`(x: TResults): string =
             "Tests skipped: $2 / $3 <br />\n") %
             [$x.passed, $x.skipped, $x.total]
 
-proc addResult(r: var TResults, test: TTest,
+proc addResult(r: var TResults, test: TTest, target: TTarget,
                expected, given: string, success: TResultEnum) =
-  let name = test.name.extractFilename & test.options
+  let name = test.name.extractFilename & " " & $target & test.options
   let duration = epochTime() - test.startTime
   backend.writeTestResult(name = name,
                           category = test.cat.string,
-                          target = $test.target,
+                          target = $target,
                           action = $test.action,
                           result = $success,
                           expected = expected,
@@ -197,29 +196,29 @@ proc addResult(r: var TResults, test: TTest,
     discard waitForExit(p)
     close(p)
 
-proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) =
+proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
   if strip(expected.msg) notin strip(given.msg):
-    r.addResult(test, expected.msg, given.msg, reMsgsDiffer)
+    r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer)
   elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg:
-    r.addResult(test, expected.nimout, given.nimout, reMsgsDiffer)
+    r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer)
   elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and
       "internal error:" notin expected.msg:
-    r.addResult(test, expected.file, given.file, reFilesDiffer)
+    r.addResult(test, target, expected.file, given.file, reFilesDiffer)
   elif expected.line   != given.line   and expected.line   != 0 or
        expected.column != given.column and expected.column != 0:
-    r.addResult(test, $expected.line & ':' & $expected.column,
+    r.addResult(test, target, $expected.line & ':' & $expected.column,
                       $given.line    & ':' & $given.column,
                       reLinesDiffer)
   elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and
       "internal error:" notin expected.msg:
-    r.addResult(test, expected.tfile, given.tfile, reFilesDiffer)
+    r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer)
   elif expected.tline   != given.tline   and expected.tline   != 0 or
        expected.tcolumn != given.tcolumn and expected.tcolumn != 0:
-    r.addResult(test, $expected.tline & ':' & $expected.tcolumn,
+    r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn,
                       $given.tline    & ':' & $given.tcolumn,
                       reLinesDiffer)
   else:
-    r.addResult(test, expected.msg, given.msg, reSuccess)
+    r.addResult(test, target, expected.msg, given.msg, reSuccess)
     inc(r.passed)
 
 proc generatedFile(path, name: string, target: TTarget): string =
@@ -231,11 +230,11 @@ proc generatedFile(path, name: string, target: TTarget): string =
 proc needsCodegenCheck(spec: TSpec): bool =
   result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
 
-proc codegenCheck(test: TTest, spec: TSpec, expectedMsg: var string,
+proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string,
                   given: var TSpec) =
   try:
     let (path, name, _) = test.name.splitFile
-    let genFile = generatedFile(path, name, test.target)
+    let genFile = generatedFile(path, name, target)
     let contents = readFile(genFile).string
     let check = spec.ccodeCheck
     if check.len > 0:
@@ -267,13 +266,13 @@ proc makeDeterministic(s: string): string =
   sort(x, system.cmp)
   result = join(x, "\n")
 
-proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec;
-                         r: var TResults) =
+proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
+                         expected: TSpec; r: var TResults) =
   var expectedmsg: string = ""
   var givenmsg: string = ""
   if given.err == reSuccess:
     if expected.needsCodegenCheck:
-      codegenCheck(test, expected, expectedmsg, given)
+      codegenCheck(test, target, expected, expectedmsg, given)
       givenmsg = given.msg
     if expected.nimout.len > 0:
       expectedmsg = expected.nimout
@@ -282,7 +281,7 @@ proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec;
   else:
     givenmsg = given.nimout.strip
   if given.err == reSuccess: inc(r.passed)
-  r.addResult(test, expectedmsg, givenmsg, given.err)
+  r.addResult(test, target, expectedmsg, givenmsg, given.err)
 
 proc analyzeAndConsolidateOutput(s: string): string =
   result = ""
@@ -298,13 +297,7 @@ proc analyzeAndConsolidateOutput(s: string): string =
       result = substr(rows[i], pos)
       return
 
-proc testSpec(r: var TResults, test: TTest) =
-  # major entry point for a single test
-  if test.target notin targets:
-    r.addResult(test, "", "", reIgnored)
-    inc(r.skipped)
-    return
-
+proc testSpec(r: var TResults, test: TTest, target = targetC) =
   let tname = test.name.addFileExt(".nim")
   #echo "TESTING ", tname
   inc(r.total)
@@ -316,83 +309,92 @@ proc testSpec(r: var TResults, test: TTest) =
     expected.action = actionRunNoSpec
 
   if expected.err == reIgnored:
-    r.addResult(test, "", "", reIgnored)
+    r.addResult(test, target, "", "", reIgnored)
     inc(r.skipped)
     return
 
-  case expected.action
-  of actionCompile:
-    var given = callCompiler(expected.cmd, test.name,
-      test.options & " --stdout --hint[Path]:off --hint[Processing]:off",
-      test.target)
-    compilerOutputTests(test, given, expected, r)
-  of actionRun, actionRunNoSpec:
-    # In this branch of code "early return" pattern is clearer than deep
-    # nested conditionals - the empty rows in between to clarify the "danger"
-    var given = callCompiler(expected.cmd, test.name, test.options,
-                             test.target)
+  if expected.targets == {}:
+    expected.targets.incl(target)
 
-    if given.err != reSuccess:
-      r.addResult(test, "", given.msg, given.err)
-      return
+  for target in expected.targets:
+    if target notin targets:
+      r.addResult(test, target, "", "", reIgnored)
+      inc(r.skipped)
+      continue
 
-    let isJsTarget = test.target == targetJS
-    var exeFile: string
-    if isJsTarget:
-      let (dir, file, _) = splitFile(tname)
-      exeFile = dir / "nimcache" / file & ".js" # *TODO* hardcoded "nimcache"
-    else:
-      exeFile = changeFileExt(tname, ExeExt)
+    case expected.action
+    of actionCompile:
+      var given = callCompiler(expected.cmd, test.name,
+        test.options & " --stdout --hint[Path]:off --hint[Processing]:off",
+        target)
+      compilerOutputTests(test, target, given, expected, r)
+    of actionRun, actionRunNoSpec:
+      # In this branch of code "early return" pattern is clearer than deep
+      # nested conditionals - the empty rows in between to clarify the "danger"
+      var given = callCompiler(expected.cmd, test.name, test.options,
+                               target)
 
-    if not existsFile(exeFile):
-      r.addResult(test, expected.outp, "executable not found", reExeNotFound)
-      return
+      if given.err != reSuccess:
+        r.addResult(test, target, "", given.msg, given.err)
+        continue
 
-    let nodejs = if isJsTarget: findNodeJs() else: ""
-    if isJsTarget and nodejs == "":
-      r.addResult(test, expected.outp, "nodejs binary not in PATH",
-                  reExeNotFound)
-      return
+      let isJsTarget = target == targetJS
+      var exeFile: string
+      if isJsTarget:
+        let (dir, file, _) = splitFile(tname)
+        exeFile = dir / "nimcache" / file & ".js" # *TODO* hardcoded "nimcache"
+      else:
+        exeFile = changeFileExt(tname, ExeExt)
 
-    let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile
-    var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut})
+      if not existsFile(exeFile):
+        r.addResult(test, target, expected.outp, "executable not found", reExeNotFound)
+        continue
 
-    # Treat all failure codes from nodejs as 1. Older versions of nodejs used
-    # to return other codes, but for us it is sufficient to know that it's not 0.
-    if exitCode != 0: exitCode = 1
+      let nodejs = if isJsTarget: findNodeJs() else: ""
+      if isJsTarget and nodejs == "":
+        r.addResult(test, target, expected.outp, "nodejs binary not in PATH",
+                    reExeNotFound)
+        continue
 
-    let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string))
-               else: strip(buf.string)
-    let expectedOut = strip(expected.outp)
+      let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile
+      var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut})
 
-    if exitCode != expected.exitCode:
-      r.addResult(test, "exitcode: " & $expected.exitCode,
-                        "exitcode: " & $exitCode & "\n\nOutput:\n" &
-                        analyzeAndConsolidateOutput(bufB),
-                        reExitCodesDiffer)
-      return
+      # Treat all failure codes from nodejs as 1. Older versions of nodejs used
+      # to return other codes, but for us it is sufficient to know that it's not 0.
+      if exitCode != 0: exitCode = 1
 
-    if bufB != expectedOut and expected.action != actionRunNoSpec:
-      if not (expected.substr and expectedOut in bufB):
-        given.err = reOutputsDiffer
-        r.addResult(test, expected.outp, bufB, reOutputsDiffer)
-        return
+      let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string))
+                 else: strip(buf.string)
+      let expectedOut = strip(expected.outp)
 
-    compilerOutputTests(test, given, expected, r)
-    return
+      if exitCode != expected.exitCode:
+        r.addResult(test, target, "exitcode: " & $expected.exitCode,
+                          "exitcode: " & $exitCode & "\n\nOutput:\n" &
+                          analyzeAndConsolidateOutput(bufB),
+                          reExitCodesDiffer)
+        continue
 
-  of actionReject:
-    var given = callCompiler(expected.cmd, test.name, test.options,
-                             test.target)
-    cmpMsgs(r, expected, given, test)
-    return
+      if bufB != expectedOut and expected.action != actionRunNoSpec:
+        if not (expected.substr and expectedOut in bufB):
+          given.err = reOutputsDiffer
+          r.addResult(test, target, expected.outp, bufB, reOutputsDiffer)
+          continue
 
-proc testNoSpec(r: var TResults, test: TTest) =
+      compilerOutputTests(test, target, given, expected, r)
+      continue
+
+    of actionReject:
+      var given = callCompiler(expected.cmd, test.name, test.options,
+                               target)
+      cmpMsgs(r, expected, given, test, target)
+      continue
+
+proc testNoSpec(r: var TResults, test: TTest, target = targetC) =
   # does not extract the spec because the file is not supposed to have any
   #let tname = test.name.addFileExt(".nim")
   inc(r.total)
-  let given = callCompiler(cmdTemplate(), test.name, test.options, test.target)
-  r.addResult(test, "", given.msg, given.err)
+  let given = callCompiler(cmdTemplate(), test.name, test.options, target)
+  r.addResult(test, target, "", given.msg, given.err)
   if given.err == reSuccess: inc(r.passed)
 
 proc testC(r: var TResults, test: TTest) =
@@ -400,9 +402,9 @@ proc testC(r: var TResults, test: TTest) =
   let tname = test.name.addFileExt(".c")
   inc(r.total)
   styledEcho "Processing ", fgCyan, extractFilename(tname)
-  var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, test.target)
+  var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
   if given.err != reSuccess:
-    r.addResult(test, "", given.msg, given.err)
+    r.addResult(test, targetC, "", given.msg, given.err)
   elif test.action == actionRun:
     let exeFile = changeFileExt(test.name, ExeExt)
     var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
@@ -410,10 +412,10 @@ proc testC(r: var TResults, test: TTest) =
   if given.err == reSuccess: inc(r.passed)
 
 proc makeTest(test, options: string, cat: Category, action = actionCompile,
-              target = targetC, env: string = ""): TTest =
+              env: string = ""): TTest =
   # start with 'actionCompile', will be overwritten in the spec:
   result = TTest(cat: cat, name: test, options: options,
-                 target: target, action: action, startTime: epochTime())
+                 action: action, startTime: epochTime())
 
 when defined(windows):
   const