make megatest consistent with unjoined tests wrt newlines, honor newlines in output spec (#16151)

* fix megatest newlines
* still allow missing trailing newline for now but in a more strict way than before

testament/categories.nim

@@ -591,12 +591,14 @@ proc isJoinableSpec(spec: TSpec): bool =
     if spec.file.readFile.contains "when isMainModule":
       result = false
 
-proc norm(s: var string) =
-  while true:
-    let tmp = s.replace("\n\n", "\n")
-    if tmp == s: break
-    s = tmp
-  s = s.strip
+when false:
+  proc norm(s: var string) =
+    ## strip empty newlines
+    while true:
+      let tmp = s.replace("\n\n", "\n")
+      if tmp == s: break
+      s = tmp
+    s = s.strip
 
 proc quoted(a: string): string =
   # todo: consider moving to system.nim
@@ -654,16 +656,16 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
     echo buf.string
     quit(failString & "megatest execution failed")
 
-  norm buf.string
   const outputExceptedFile = "outputExpected.txt"
   const outputGottenFile = "outputGotten.txt"
   writeFile(outputGottenFile, buf.string)
   var outputExpected = ""
   for i, runSpec in specs:
     outputExpected.add marker & runSpec.file & "\n"
-    outputExpected.add runSpec.output.strip
-    outputExpected.add '\n'
-  norm outputExpected
+    if runSpec.output.len > 0:
+      outputExpected.add runSpec.output
+      if not runSpec.output.endsWith "\n":
+        outputExpected.add '\n'
 
   if buf.string != outputExpected:
     writeFile(outputExceptedFile, outputExpected)

testament/specs.nim

@@ -280,7 +280,7 @@ proc parseSpec*(filename: string): TSpec =
       of "output":
         if result.outputCheck != ocSubstr:
           result.outputCheck = ocEqual
-        result.output = strip(e.value)
+        result.output = e.value
       of "input":
         result.input = e.value
       of "outputsub":

testament/testament.nim

@@ -469,6 +469,10 @@ proc checkDisabled(r: var TResults, test: TTest): bool =
 
 var count = 0
 
+proc equalModuloLastNewline(a, b: string): bool =
+  # allow lazy output spec that omits last newline, but really those should be fixed instead
+  result = a == b or b.endsWith("\n") and a == b[0 ..< ^1]
+
 proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
                     target: TTarget, nimcache: string, extraOptions = "") =
   test.startTime = epochTime()
@@ -513,16 +517,18 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
           if exitCode != 0: exitCode = 1
           let bufB =
             if expected.sortoutput:
-              var x = splitLines(strip(buf.string))
+              var buf2 = buf.string
+              buf2.stripLineEnd
+              var x = splitLines(buf2)
               sort(x, system.cmp)
-              join(x, "\n")
+              join(x, "\n") & "\n"
             else:
-              strip(buf.string)
+              buf.string
           if exitCode != expected.exitCode:
             r.addResult(test, target, "exitcode: " & $expected.exitCode,
                               "exitcode: " & $exitCode & "\n\nOutput:\n" &
                               bufB, reExitcodesDiffer)
-          elif (expected.outputCheck == ocEqual and expected.output != bufB) or
+          elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or
               (expected.outputCheck == ocSubstr and expected.output notin bufB):
             given.err = reOutputsDiffer
             r.addResult(test, target, expected.output, bufB, reOutputsDiffer)

tests/arc/tstrformat.nim

@@ -1,5 +1,7 @@
 discard """
-  output: '''verstuff'''
+  output: '''
+verstuff
+'''
   cmd: "nim c --gc:arc $file"
 """
 

tests/async/tasyncfilewrite.nim

@@ -1,7 +1,8 @@
 discard """
   output: '''string 1
 string 2
-string 3'''
+string 3
+'''
 """
 # bug #5532
 import os, asyncfile, asyncdispatch

tests/ccgbugs/tret_arg_init.nim

@@ -1,6 +1,8 @@
 discard """
   output: '''
 
+
+
 '''
 """
 

tests/concepts/treversable.nim

@@ -3,7 +3,7 @@ discard """
   output: '''
 z
 e
-  '''
+'''
 """
 
 type

tests/destructor/ttuple.nim

@@ -2,7 +2,7 @@
 discard """
    output: '''5.0 10.0
 =destroy
-=destroy   
+=destroy
 '''
 """
 

tests/exception/texcas.nim

@@ -1,8 +1,9 @@
 discard """
   targets: "c cpp"
-  output: '''Hello
+  output: '''
 Hello
-  '''
+Hello
+'''
 """
 proc test[T]() =
   try:

tests/exception/texceptions.nim

@@ -1,5 +1,6 @@
 discard """
   output: '''
+
 BEFORE
 FINALLY
 

tests/generics/treentranttypes.nim

@@ -2,21 +2,13 @@ discard """
 output: '''
 (10, ("test", 1.2))
 3x3 Matrix [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0], [2.0, 0.0, 5.0]]
-
 2x3 Matrix [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0]]
-
 2x3 Literal [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0]]
-
 2x3 Matrix [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
-
 2x2 ArrayArray[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
-
 2x3 ArrayVector[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
-
 2x3 VectorVector [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
-
 2x3 VectorArray [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
-
 @[1, 2]
 @[1, 2]
 @[1, 2]@[3, 4]

tests/js/tunittests.nim

@@ -1,7 +1,9 @@
 discard """
   output: '''
+
 [Suite] Bacon
-  [OK] >:)'''
+  [OK] >:)
+'''
 """
 
 import unittest

tests/lookups/test.nim

@@ -1,7 +1,7 @@
 discard """
 output: '''
-[Suite] memoization
 
+[Suite] memoization
 '''
 """
 

tests/macros/tastrepr.nim

@@ -1,5 +1,6 @@
 discard """
 output: '''
+
 var data = @[(1, "one"), (2, "two")]
 for (i, d) in pairs(data):
   discard

tests/magics/t10307.nim

@@ -4,7 +4,8 @@ discard """
 res: yes
 yes
 running someProc(false)
-res:
+res: 
+
 '''
 """
 
@@ -20,4 +21,4 @@ echo "running someProc(true)"
 echo someProc(true)
 
 echo "running someProc(false)"
-echo someProc(false)
+echo someProc(false)

tests/metatype/tmatrix3.nim

@@ -1,5 +1,5 @@
 discard """
-  output: ""
+  output: "\n"
 """
 
 type Matrix[M,N: static[int]] = array[M, array[N, float]]

tests/metatype/ttensor.nim

@@ -1,13 +1,11 @@
 discard """
-  output: '''before tensor2:
+  output: '''
+before tensor2:
 [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0]
-
 before tensor3:
 [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0]
-
 after tensor3:
 [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0]
-
 a3:
 [1000.0, 1001.0, 1002.0, 1003.0, 1004.0, 1005.0, 1006.0, 1007.0, 1008.0, 1009.0, 1010.0, 1011.0, 1012.0, 1013.0, 1014.0, 1015.0, 1016.0, 1017.0, 1018.0, 1019.0, 1020.0, 1021.0, 1022.0, 1023.0, 1024.0, 1025.0, 1026.0]'''
 """

tests/misc/temptyecho.nim

@@ -1,2 +1,6 @@
+discard """
+output: "\n"
+"""
+
 echo()
 

tests/objects/tobject.nim

@@ -1,5 +1,5 @@
 discard """
-output: "[Suite] object basic methods"
+output: "\n[Suite] object basic methods"
 """
 
 import unittest

tests/overload/tstatic_with_converter.nim

@@ -1,6 +1,8 @@
 discard """
 output: '''
-9.0'''
+9.0
+
+'''
 """
 
 ### bug #6773
@@ -41,6 +43,7 @@ proc `^`(x: vfloat, exp: static[float]): vfloat =
  
 proc `$`(x: vfloat): string =
   let y = cast[ptr float](unsafeAddr x)
+  # xxx not sure if intentional in this issue, but this returns ""
   echo y[]
  
 let x = set1(9.0)

tests/specialops/tdotops.nim

@@ -1,5 +1,6 @@
 discard """
   output: '''
+
 10
 assigning z = 20
 reading field y
@@ -12,7 +13,8 @@ no params call to b
 100
 one param call to c with 10
 100
-0 4'''
+0 4
+'''
 """
 
 type

tests/statictypes/tpassthruarith.nim

@@ -1,13 +1,9 @@
 discard """
 output: '''
 [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
-
 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
-
 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
-
 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
-
 [1, 2, 3, 4]
 '''
 """

tests/stdlib/talgorithm.nim

@@ -1,6 +1,6 @@
 discard """
   output:'''@["3", "2", "1"]
-  '''
+'''
 """
 #12928,10456
 import sequtils, strutils, algorithm, json

tests/stdlib/tcgi.nim

@@ -1,5 +1,6 @@
 discard """
   output: '''
+
 [Suite] Test cgi module
 (key: "a", value: "1")
 (key: "b", value: "0")

tests/stdlib/tmath.nim

@@ -1,6 +1,8 @@
 discard """
   action: run
-  output: '''[Suite] random int
+  output: '''
+
+[Suite] random int
 
 [Suite] random float
 
@@ -9,7 +11,6 @@ discard """
 [Suite] random sample
 
 [Suite] ^
-
 '''
 """
 

tests/stdlib/tnet_ll.nim

@@ -1,6 +1,7 @@
 discard """
   action: run
   output: '''
+
 [Suite] inet_ntop tests
 '''
 """

tests/stdlib/tnre.nim

@@ -4,7 +4,9 @@ discard """
 # codes to differ and be reported as a failure
 
   output:
-    '''[Suite] Test NRE initialization
+    '''
+
+[Suite] Test NRE initialization
 
 [Suite] captures
 

tests/stdlib/tparseuints.nim

@@ -1,13 +1,14 @@
 discard """
   action: run
   output: '''
-[Suite] parseutils'''
+
+[Suite] parseutils
+'''
 """
 import unittest, strutils
 
 suite "parseutils":
-  test "uint":
-    check: parseBiggestUInt("0") == 0'u64
-    check: parseBiggestUInt("18446744073709551615") == 0xFFFF_FFFF_FFFF_FFFF'u64
-    expect(ValueError):
-      discard parseBiggestUInt("18446744073709551616")
+  check: parseBiggestUInt("0") == 0'u64
+  check: parseBiggestUInt("18446744073709551615") == 0xFFFF_FFFF_FFFF_FFFF'u64
+  expect(ValueError):
+    discard parseBiggestUInt("18446744073709551616")

tests/stdlib/trst.nim

@@ -1,5 +1,6 @@
 discard """
   output: '''
+
 [Suite] RST include directive
 '''
 """

tests/stdlib/tsequtils.nim

@@ -455,6 +455,3 @@ block:
         yield i
 
   doAssert: iter(3).mapIt(2*it).foldl(a + b) == 6
-
-when not defined(testing):
-  echo "Finished doc tests"

tests/stdlib/tunittest.nim

@@ -1,5 +1,7 @@
 discard """
-  output: '''[Suite] suite with only teardown
+  output: '''
+
+[Suite] suite with only teardown
 
 [Suite] suite with only setup
 
@@ -16,7 +18,6 @@ discard """
 [Suite] test suite
 
 [Suite] test name filtering
-
 '''
 """
 
@@ -141,38 +142,37 @@ suite "test suite":
 
         check(a == b)
 
-when defined(testing):
-  suite "test name filtering":
-    test "test name":
-      check matchFilter("suite1", "foo", "")
-      check matchFilter("suite1", "foo", "foo")
-      check matchFilter("suite1", "foo", "::")
-      check matchFilter("suite1", "foo", "*")
-      check matchFilter("suite1", "foo", "::foo")
-      check matchFilter("suite1", "::foo", "::foo")
+suite "test name filtering":
+  test "test name":
+    check matchFilter("suite1", "foo", "")
+    check matchFilter("suite1", "foo", "foo")
+    check matchFilter("suite1", "foo", "::")
+    check matchFilter("suite1", "foo", "*")
+    check matchFilter("suite1", "foo", "::foo")
+    check matchFilter("suite1", "::foo", "::foo")
 
-    test "test name - glob":
-      check matchFilter("suite1", "foo", "f*")
-      check matchFilter("suite1", "foo", "*oo")
-      check matchFilter("suite1", "12345", "12*345")
-      check matchFilter("suite1", "q*wefoo", "q*wefoo")
-      check false == matchFilter("suite1", "foo", "::x")
-      check false == matchFilter("suite1", "foo", "::x*")
-      check false == matchFilter("suite1", "foo", "::*x")
-      #  overlap
-      check false == matchFilter("suite1", "12345", "123*345")
-      check matchFilter("suite1", "ab*c::d*e::f", "ab*c::d*e::f")
+  test "test name - glob":
+    check matchFilter("suite1", "foo", "f*")
+    check matchFilter("suite1", "foo", "*oo")
+    check matchFilter("suite1", "12345", "12*345")
+    check matchFilter("suite1", "q*wefoo", "q*wefoo")
+    check false == matchFilter("suite1", "foo", "::x")
+    check false == matchFilter("suite1", "foo", "::x*")
+    check false == matchFilter("suite1", "foo", "::*x")
+    #  overlap
+    check false == matchFilter("suite1", "12345", "123*345")
+    check matchFilter("suite1", "ab*c::d*e::f", "ab*c::d*e::f")
 
-    test "suite name":
-      check matchFilter("suite1", "foo", "suite1::")
-      check false == matchFilter("suite1", "foo", "suite2::")
-      check matchFilter("suite1", "qwe::foo", "qwe::foo")
-      check matchFilter("suite1", "qwe::foo", "suite1::qwe::foo")
+  test "suite name":
+    check matchFilter("suite1", "foo", "suite1::")
+    check false == matchFilter("suite1", "foo", "suite2::")
+    check matchFilter("suite1", "qwe::foo", "qwe::foo")
+    check matchFilter("suite1", "qwe::foo", "suite1::qwe::foo")
 
-    test "suite name - glob":
-      check matchFilter("suite1", "foo", "::*")
-      check matchFilter("suite1", "foo", "*::*")
-      check matchFilter("suite1", "foo", "*::foo")
-      check false == matchFilter("suite1", "foo", "*ite2::")
-      check matchFilter("suite1", "q**we::foo", "q**we::foo")
-      check matchFilter("suite1", "a::b*c::d*e", "a::b*c::d*e")
+  test "suite name - glob":
+    check matchFilter("suite1", "foo", "::*")
+    check matchFilter("suite1", "foo", "*::*")
+    check matchFilter("suite1", "foo", "*::foo")
+    check false == matchFilter("suite1", "foo", "*ite2::")
+    check matchFilter("suite1", "q**we::foo", "q**we::foo")
+    check matchFilter("suite1", "a::b*c::d*e", "a::b*c::d*e")

tests/system/tenum_array_repr.nim

@@ -2,10 +2,8 @@ discard """
   output: '''
 1
 [a, b]
-
 2
 [c, d]
-
 4
 [e, f]'''
 """

tests/template/tgensymregression.nim

@@ -1,8 +1,6 @@
 discard """
   output: '''[0.0, 0.0, 0.0]
-
 [0.0, 0.0, 0.0, 0.0]
-
 5050
 123'''
 """