[Testament] Extend and document message testing aids (#19996)

* [Testament] Extend and document message testing aids * Enable inline msgs when not reject action. Eliminates the pain of changing the line and column numbers in `nimout` or `output` while making changes to the test. * Enable using inline msgs and nimout together. Allows ease of inline msgs for the test as well as testing msgs from other modules. * Add path separator and test filename variable interpolation in msgs. Eases handling path separators in the msgs. * Add some documentation. * Fixed lots of broken tests * Fixed more broken tests * Support multiple inline messages per a line * Fix a broken test * Revert variable substitution in `output` * Remove uneeded params * Update doc/testament.md Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com> * Update testament/specs.nim Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com> * Update testament/specs.nim Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com> * Fix indentation Co-authored-by: quantimnot <quantimnot@users.noreply.github.com> Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>
2026-01-21 03:50:43 +00:00 · 2022-09-01 11:52:13 -04:00
parent c2cdc752c8
commit 6289b002b6
15 changed files with 247 additions and 149 deletions
--- a/doc/testament.md
+++ b/doc/testament.md
@@ -113,7 +113,7 @@ Example "template" **to edit** and write a Testament unittest:
    # Provide an `output` string to assert that the test prints to standard out
    # exactly the expected string. Provide an `outputsub` string to assert that
    # the string given here is a substring of the standard out output of the
-    # test.
+    # test (the output includes both the compiler and test execution output).
    output: ""
    outputsub: ""

@@ -153,8 +153,7 @@ Example "template" **to edit** and write a Testament unittest:
    # Command the test should use to run. If left out or an empty string is
    # provided, the command is taken to be:
    # "nim $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file"
-    # You can use the $target, $options, and $file placeholders in your own
-    # command, too.
+    # Subject to variable interpolation.
    cmd: "nim c -r $file"

    # Maximum generated temporary intermediate code file size for the test.
@@ -189,7 +188,76 @@ Example "template" **to edit** and write a Testament unittest:
 * `This is not the full spec of Testament, check the Testament Spec on GitHub, see parseSpec(). <https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L238>`_
 * `Nim itself uses Testament, so there are plenty of test examples. <https://github.com/nim-lang/Nim/tree/devel/tests>`_
 * Has some built-in CI compatibility, like Azure Pipelines, etc.
-* `Testament supports inlined error messages on Unittests, basically comments with the expected error directly on the code. <https://github.com/nim-lang/Nim/blob/9a110047cbe2826b1d4afe63e3a1f5a08422b73f/tests/effects/teffects1.nim>`_
+
+
+Inline hints, warnings and errors (notes)
+-----------------------------------------
+
+Testing the line, column, kind and message of hints, warnings and errors can
+be written inline like so:
+
+.. code-block:: nim
+
+  {.warning: "warning!!"} #[tt.Warning
+           ^ warning!! [User] ]#
+
+The opening `#[tt.` marks the message line.
+The `^` marks the message column.
+
+Inline messages can be combined with `nimout` when `nimoutFull` is false (default).
+This allows testing for expected messages from other modules:
+
+.. code-block:: nim
+
+  discard """
+    nimout: "config.nims(1, 1) Hint: some hint message [User]"
+  """
+  {.warning: "warning!!"} #[tt.Warning
+           ^ warning!! [User] ]#
+
+Multiple messages for a line can be checked by delimiting messages with ';':
+
+.. code-block:: nim
+
+  discard """
+    matrix: "--errorMax:0 --styleCheck:error"
+  """
+
+  proc generic_proc*[T](a_a: int) = #[tt.Error
+       ^ 'generic_proc' should be: 'genericProc'; tt.Error
+                        ^ 'a_a' should be: 'aA' ]#
+    discard
+
+Use `--errorMax:0` in `matrix`, or `cmd: "nim check $file"` when testing
+for multiple 'Error' messages.
+
+Output message variable interpolation
+-------------------------------------
+
+`errormsg`, `nimout`, and inline messages are subject to these variable interpolations:
+
+* `${/}` - platform's directory separator
+* `$file` - the filename (without directory) of the test
+
+All other `$` characters need escaped as `$$`.
+
+Cmd variable interpolation
+--------------------------
+
+The `cmd` option is subject to these variable interpolations:
+
+* `$target` - the compilation target, e.g. `c`.
+* `$options` - the options for the compiler.
+* `$file` - the file path of the test.
+* `$filedir` - the directory of the test file.
+
+.. code-block:: nim
+
+  discard """
+    cmd: "nim $target --nimblePath:./nimbleDir/simplePkgs $options $file"
+  """
+
+All other `$` characters need escaped as `$$`.


 Unit test Examples
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -75,6 +75,7 @@ type
    # xxx make sure `isJoinableSpec` takes into account each field here.
    action*: TTestAction
    file*, cmd*: string
+    filename*: string ## Test filename (without path).
    input*: string
    outputCheck*: TOutputCheck
    sortoutput*: bool
@@ -127,19 +128,56 @@ when not declared(parseCfgBool):
    of "n", "no", "false", "0", "off": result = false
    else: raise newException(ValueError, "cannot interpret as a bool: " & s)

+proc addLine*(self: var string; pieces: varargs[string]) =
+  for piece in pieces:
+    self.add piece
+  self.add "\n"
+
+
 const
-  inlineErrorMarker = "#[tt."
+  inlineErrorKindMarker = "tt."
+  inlineErrorMarker = "#[" & inlineErrorKindMarker

 proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int =
+  ## Extract inline error messages.
+  ##
+  ## Can parse a single message for a line:
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name] ]#
+  ##
+  ## Can parse multiple messages for a line when they are separated by ';':
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error
+  ##                           ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error
+  ##                                       ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name];
+  ##     tt.Error              ^ 'no_destroy' should be: 'nodestroy' [Name];
+  ##     tt.Error                          ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##
  result = i + len(inlineErrorMarker)
  inc col, len(inlineErrorMarker)
+  let msgLine = line
+  var msgCol = -1
+  var msg = ""
  var kind = ""
-  while result < s.len and s[result] in IdentChars:
-    kind.add s[result]
-    inc result
-    inc col

-  var caret = (line, -1)
+  template parseKind =
+    while result < s.len and s[result] in IdentChars:
+      kind.add s[result]
+      inc result
+      inc col
+    if kind notin ["Hint", "Warning", "Error"]:
+      spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error"

  template skipWhitespace =
    while result < s.len and s[result] in Whitespace:
@@ -150,34 +188,70 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T
        inc col
      inc result

-  skipWhitespace()
-  if result < s.len and s[result] == '^':
-    caret = (line-1, col)
-    inc result
-    inc col
-    skipWhitespace()
+  template parseCaret =
+    if result < s.len and s[result] == '^':
+      msgCol = col
+      inc result
+      inc col
+      skipWhitespace()
+    else:
+      spec.parseErrors.addLine "expected column marker ('^') for inline message"
+
+  template isMsgDelimiter: bool =
+    s[result] == ';' and
+    (block:
+      let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1)
+      if s.len > nextTokenIdx + len(inlineErrorKindMarker) and
+         s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker:
+        true
+      else:
+        false)
+
+  template trimTrailingMsgWhitespace =
+    while msg.len > 0 and msg[^1] in Whitespace:
+      setLen msg, msg.len - 1
+
+  template addInlineError =
+    doAssert msg[^1] notin Whitespace
+    if kind == "Error": spec.action = actionReject
+    spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol)
+
+  parseKind()
+  skipWhitespace()
+  parseCaret()

-  var msg = ""
  while result < s.len-1:
    if s[result] == '\n':
+      msg.add '\n'
      inc result
      inc line
      col = 1
-    elif s[result] == ']' and s[result+1] == '#':
-      while msg.len > 0 and msg[^1] in Whitespace:
-        setLen msg, msg.len - 1
-
+    elif isMsgDelimiter():
+      trimTrailingMsgWhitespace()
      inc result
+      skipWhitespace()
+      addInlineError()
+      inc result, len(inlineErrorKindMarker)
+      inc col, 1 + len(inlineErrorKindMarker)
+      kind.setLen 0
+      msg.setLen 0
+      parseKind()
+      skipWhitespace()
+      parseCaret()
+    elif s[result] == ']' and s[result+1] == '#':
+      trimTrailingMsgWhitespace()
+      inc result, 2
      inc col, 2
-      if kind == "Error": spec.action = actionReject
-      spec.unjoinable = true
-      spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: caret[0], col: caret[1])
+      addInlineError()
      break
    else:
      msg.add s[result]
      inc result
      inc col

+  if spec.inlineErrors.len > 0:
+    spec.unjoinable = true
+
 proc extractSpec(filename: string; spec: var TSpec): string =
  const
    tripleQuote = "\"\"\""
@@ -229,15 +303,6 @@ proc parseTargets*(value: string): set[TTarget] =
    of "js": result.incl(targetJS)
    else: raise newException(ValueError, "invalid target: '$#'" % v)

-proc addLine*(self: var string; a: string) =
-  self.add a
-  self.add "\n"
-
-proc addLine*(self: var string; a, b: string) =
-  self.add a
-  self.add b
-  self.add "\n"
-
 proc initSpec*(filename: string): TSpec =
  result.file = filename

@@ -249,6 +314,7 @@ proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool =

 proc parseSpec*(filename: string): TSpec =
  result.file = filename
+  result.filename = extractFilename(filename)
  let specStr = extractSpec(filename, result)
  var ss = newStringStream(specStr)
  var p: CfgParser
@@ -439,3 +505,15 @@ proc parseSpec*(filename: string): TSpec =
  result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
  if not result.inCurrentBatch:
    result.err = reDisabled
+
+  # Interpolate variables in msgs:
+  template varSub(msg: string): string =
+    try:
+      msg % ["/", $DirSep, "file", result.filename]
+    except ValueError:
+      result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unitests-output-message-variable-interpolation')"
+      msg
+  result.nimout = result.nimout.varSub
+  result.msg = result.msg.varSub
+  for inlineError in result.inlineErrors.mitems:
+    inlineError.msg = inlineError.msg.varSub
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -338,46 +338,24 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
      discard waitForExit(p)
      close(p)

-proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest,
-                          target: TTarget, extraOptions: string) =
-  let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}"
-  var covered = initIntSet()
-  for line in splitLines(given.nimout):
+proc toString(inlineError: InlineError, filename: string): string =
+  result.add "$file($line, $col) $kind: $msg" % [
+    "file", filename,
+    "line", $inlineError.line,
+    "col", $inlineError.col,
+    "kind", $inlineError.kind,
+    "msg", $inlineError.msg
+  ]

-    if line =~ pegLine:
-      let file = extractFilename(matches[0])
-      let line = try: parseInt(matches[1]) except: -1
-      let col = try: parseInt(matches[2]) except: -1
-      let kind = matches[3]
-      let msg = matches[4]
+proc inlineErrorsMsgs(expected: TSpec): string =
+  for inlineError in expected.inlineErrors.items:
+    result.addLine inlineError.toString(expected.filename)

-      if file == extractFilename test.name:
-        var i = 0
-        for x in expected.inlineErrors:
-          if x.line == line and (x.col == col or x.col < 0) and
-              x.kind == kind and x.msg in msg:
-            covered.incl i
-          inc i
-
-  block coverCheck:
-    for j in 0..high(expected.inlineErrors):
-      if j notin covered:
-        var e = test.name
-        e.add '('
-        e.addInt expected.inlineErrors[j].line
-        if expected.inlineErrors[j].col > 0:
-          e.add ", "
-          e.addInt expected.inlineErrors[j].col
-        e.add ") "
-        e.add expected.inlineErrors[j].kind
-        e.add ": "
-        e.add expected.inlineErrors[j].msg
-
-        r.addResult(test, target, extraOptions, e, given.nimout, reMsgsDiffer)
-        break coverCheck
-
-    r.addResult(test, target, extraOptions, "", given.msg, reSuccess)
-    inc(r.passed)
+proc checkForInlineErrors(expected, given: TSpec): bool =
+  for inlineError in expected.inlineErrors:
+    if inlineError.toString(expected.filename) notin given.nimout:
+      return false
+  true

 proc nimoutCheck(expected, given: TSpec): bool =
  result = true
@@ -389,15 +367,16 @@ proc nimoutCheck(expected, given: TSpec): bool =

 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
             target: TTarget, extraOptions: string) =
-  if expected.inlineErrors.len > 0:
-    checkForInlineErrors(r, expected, given, test, target, extraOptions)
+  if not checkForInlineErrors(expected, given) or
+    (not expected.nimoutFull and not nimoutCheck(expected, given)):
+      r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer)
  elif strip(expected.msg) notin strip(given.msg):
    r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
  elif not nimoutCheck(expected, given):
    r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
  elif extractFilename(expected.file) != extractFilename(given.file) and
      "internal error:" notin expected.msg:
-    r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
+    r.addResult(test, target, extraOptions, expected.filename, given.file, reFilesDiffer)
  elif expected.line != given.line and expected.line != 0 or
       expected.column != given.column and expected.column != 0:
    r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column,
@@ -449,9 +428,10 @@ proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
    if expected.needsCodegenCheck:
      codegenCheck(test, target, expected, expectedmsg, given)
      givenmsg = given.msg
-    if not nimoutCheck(expected, given):
+    if not nimoutCheck(expected, given) or
+       not checkForInlineErrors(expected, given):
      given.err = reMsgsDiffer
-      expectedmsg = expected.nimout
+      expectedmsg = expected.nimout & inlineErrorsMsgs(expected)
      givenmsg = given.nimout.strip
  else:
    givenmsg = "$ " & given.cmd & '\n' & given.nimout
@@ -480,17 +460,11 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
    r.addResult(test, target, extraOptions, "", "", test.spec.err)
    inc(r.skipped)
    return
-
-  template callNimCompilerImpl(): untyped =
-    # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently
-    # with other branches
-    callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
+  var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
  case expected.action
  of actionCompile:
-    var given = callNimCompilerImpl()
    compilerOutputTests(test, target, extraOptions, given, expected, r)
  of actionRun:
-    var given = callNimCompilerImpl()
    if given.err != reSuccess:
      r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
    else:
@@ -540,10 +514,8 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
              (expected.outputCheck == ocSubstr and expected.output notin bufB):
            given.err = reOutputsDiffer
            r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
-          else:
-            compilerOutputTests(test, target, extraOptions, given, expected, r)
+          compilerOutputTests(test, target, extraOptions, given, expected, r)
  of actionReject:
-    let given = callNimCompilerImpl()
    cmpMsgs(r, expected, given, test, target, extraOptions)

 proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =
--- a/tests/arc/topt_cursor.nim
+++ b/tests/arc/topt_cursor.nim
@@ -12,7 +12,7 @@ try:
    x_cursor = ("different", 54) else:
    x_cursor = ("string here", 80)
  echo [
-    :tmpD = `$`(x_cursor)
+    :tmpD = `$$`(x_cursor)
    :tmpD]
 finally:
  `=destroy`(:tmpD)
--- a/tests/arc/topt_no_cursor.nim
+++ b/tests/arc/topt_no_cursor.nim
@@ -78,7 +78,7 @@ try:
    `=copy`(:tmpD_1, it_cursor.val)
    :tmpD_1)
  echo [
-    :tmpD_2 = `$`(a)
+    :tmpD_2 = `$$`(a)
    :tmpD_2]
 finally:
  `=destroy`(:tmpD_2)
--- a/tests/casestmt/tincompletecaseobject2.nim
+++ b/tests/casestmt/tincompletecaseobject2.nim
@@ -1,12 +1,5 @@
 discard """
 cmd: "nim check $file"
-errormsg: "not all cases are covered; missing: {A, B}"
-nimout: '''
-tincompletecaseobject2.nim(18, 1) Error: not all cases are covered; missing: {' ', '!', '\"', '#', '$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~'}
-tincompletecaseobject2.nim(22, 1) Error: not all cases are covered; missing: {B, C, D}
-tincompletecaseobject2.nim(25, 1) Error: not all cases are covered; missing: {A, C}
-tincompletecaseobject2.nim(28, 1) Error: not all cases are covered; missing: {A, B}
-'''
 """
 type
  ABCD = enum A, B, C, D
@@ -15,15 +8,19 @@ type
  AliasRangeABC = RangeABC
  PrintableChars = range[' ' .. '~']

-case PrintableChars 'x':
+case PrintableChars 'x': #[tt.Error
+^ not all cases are covered; missing: {' ', '!', '\"', '#', '$$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~'}]#
 of '0'..'9', 'A'..'Z', 'a'..'z': discard
 of '(', ')': discard

-case AliasABCD A:
+case AliasABCD A: #[tt.Error
+^ not all cases are covered; missing: {B, C, D}]#
 of A: discard

-case RangeABC A:
+case RangeABC A: #[tt.Error
+^ not all cases are covered; missing: {A, C}]#
 of B: discard

-case AliasRangeABC A:
+case AliasRangeABC A: #[tt.Error
+^ not all cases are covered; missing: {A, B}]#
 of C: discard
--- a/tests/effects/teffects1.nim
+++ b/tests/effects/teffects1.nim
@@ -1,10 +1,8 @@
 discard """
  cmd: "nim check $file"
-  nimout: '''teffects1.nim(22, 28) template/generic instantiation from here
-teffects1.nim(23, 13) Error: can raise an unlisted exception: ref IOError
-teffects1.nim(22, 29) Hint: 'lier' cannot raise 'IO2Error' [XCannotRaiseY]
-teffects1.nim(38, 21) Error: type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'
-.raise effects differ'''
+  nimout: '''
+teffects1.nim(17, 28) template/generic instantiation from here
+'''
 """
 {.push warningAsError[Effect]: on.}
 type
@@ -16,11 +14,10 @@ type

 proc forw: int {. .}

-proc lier(): int {.raises: [IO2Error].} =
-  #[tt.Hint                 ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]#
+proc lier(): int {.raises: [IO2Error].} = #[tt.Hint
+                            ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]#
  writeLine stdout, "arg" #[tt.Error
-            ^  can raise an unlisted exception: ref IOError
-  ]#
+            ^ writeLine stdout, ["arg"] can raise an unlisted exception: ref IOError ]#

 proc forw: int =
  raise newException(IOError, "arg")
@@ -38,7 +35,7 @@ proc foo(x: int): string {.raises: [ValueError].} =
 var p: MyProcType = foo #[tt.Error
                    ^
 type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'
-
+.raise effects differ
 ]#
 {.pop.}
 {.pop.}
--- a/tests/errmsgs/tinteger_literals.nim
+++ b/tests/errmsgs/tinteger_literals.nim
@@ -1,15 +1,14 @@
 discard """
-cmd: "nim check $file"
-errormsg: "number out of range: '300'u8'"
-nimout: '''
-tinteger_literals.nim(12, 9) Error: number out of range: '18446744073709551616'u64'
-tinteger_literals.nim(13, 9) Error: number out of range: '9223372036854775808'i64'
-tinteger_literals.nim(14, 9) Error: number out of range: '9223372036854775808'
-tinteger_literals.nim(15, 9) Error: number out of range: '300'u8'
-'''
+  cmd: "nim check $file"
 """
-
-discard 18446744073709551616'u64 # high(uint64) + 1
-discard 9223372036854775808'i64  # high(int64) + 1
-discard 9223372036854775808      # high(int64) + 1
-discard 300'u8
+# high(uint64) + 1
+discard 18446744073709551616'u64 #[tt.Error
+        ^ number out of range: '18446744073709551616'u64' ]#
+# high(int64) + 1
+discard 9223372036854775808'i64 #[tt.Error
+        ^ number out of range: '9223372036854775808'i64' ]#
+# high(int64) + 1
+discard 9223372036854775808 #[tt.Error
+        ^ number out of range: '9223372036854775808' ]#
+discard 300'u8 #[tt.Error
+        ^ number out of range: '300'u8' ]#
--- a/tests/pragmas/tvar_macro.nim
+++ b/tests/pragmas/tvar_macro.nim
@@ -85,7 +85,7 @@ block: # with other pragmas
  let importedFooBar {.importc: "exportedFooBar", nodecl.}: set[char]

  doAssert importedFooBar == fooBar #[tt.Warning
-                            ^ fooBar is deprecated
+                             ^ fooBar is deprecated
  ]#
  

--- a/tests/readme.md
+++ b/tests/readme.md
@@ -27,25 +27,6 @@ Options:
 There are certain spec keys that imply ``run``, including ``output`` and
 ``outputsub``.

-## cmd
-
-Specifies the Nim command to use for compiling the test.
-
-There are a number of variables that are replaced in this spec option:
-
-* ``$target`` - the compilation target, e.g. ``c``.
-* ``$options`` - the options for the compiler.
-* ``$file`` - the filename of the test.
-* ``$filedir`` - the directory of the test file.
-
-Example:
-
-```nim
-discard """
-  cmd: "nim $target --nimblePath:./nimbleDir/simplePkgs $options $file"
-"""
-```
-
 # Categories

 Each folder under this directory represents a test category, which can be
--- a/tests/stdlib/t8925.nim
+++ b/tests/stdlib/t8925.nim
@@ -1,5 +1,5 @@
 discard """
-  errormsg: "type mismatch between pattern '$i' (position: 1) and HourRange var 'hour'"
+  errormsg: "type mismatch between pattern '$$i' (position: 1) and HourRange var 'hour'"
  file: "strscans.nim"
 """

--- a/tests/strictnotnil/tnilcheck.nim
+++ b/tests/strictnotnil/tnilcheck.nim
@@ -1,6 +1,5 @@
 discard """
-cmd: "nim check $file"
-action: "reject"
+action: compile
 """

 import tables
--- a/tests/testament/tinlinemsg.nim
+++ b/tests/testament/tinlinemsg.nim
@@ -0,0 +1,8 @@
+discard """
+  matrix: "--errorMax:0 --styleCheck:error"
+"""
+
+proc generic_proc*[T](a_a: int) = #[tt.Error
+     ^ 'generic_proc' should be: 'genericProc'; tt.Error
+                      ^ 'a_a' should be: 'aA' ]#
+  discard
--- a/tests/testament/tshould_not_work.nim
+++ b/tests/testament/tshould_not_work.nim
@@ -43,8 +43,7 @@ import stdtest/testutils

 proc main =
  const nim = getCurrentCompilerExe()
-  # TODO: bin/testament instead? like other tools (eg bin/nim, bin/nimsuggest etc)
-  let testamentExe = "testament/testament"
+  let testamentExe = "bin/testament"
  let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail"
  let (outp, status) = execCmdEx(cmd)
  doAssert status == 1, $status
--- a/tests/views/tdont_mutate.nim
+++ b/tests/views/tdont_mutate.nim
@@ -10,7 +10,7 @@ const
  Whitespace = {' ', '\t', '\n', '\r'}

 proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): Table[int, openArray[char]] #[tt.Error
-      'result' borrows from the immutable location 's' and attempts to mutate it
+^ 'result' borrows from the immutable location 's' and attempts to mutate it
    ]# =
  var last = 0
  var splits = maxsplit
@@ -35,7 +35,7 @@ proc `$`(x: openArray[char]): string =

 proc otherTest(x: int) =
  var y: var int = x #[tt.Error
-    'y' borrows from the immutable location 'x' and attempts to mutate it
+      ^ 'y' borrows from the immutable location 'x' and attempts to mutate it
  ]#
  y = 3