Testament: refactoring; makes the test joiner green

This commit is contained in:
Araq
2018-12-06 16:37:56 +01:00
parent 8896770f8f
commit 3b783f7e13
28 changed files with 176 additions and 230 deletions

View File

@@ -386,6 +386,17 @@ proc warningStr*(c: CfgParser, msg: string): string {.rtl, extern: "npc$1".} =
result = `%`("$1($2, $3) Warning: $4",
[c.filename, $getLine(c), $getColumn(c), msg])
proc ignoreMsg*(c: CfgParser, e: CfgEvent): string {.rtl, extern: "npc$1".} =
## returns a properly formatted warning message containing that
## an entry is ignored.
case e.kind
of cfgSectionStart: result = c.warningStr("section ignored: " & e.section)
of cfgKeyValuePair: result = c.warningStr("key ignored: " & e.key)
of cfgOption:
result = c.warningStr("command ignored: " & e.key & ": " & e.value)
of cfgError: result = e.msg
of cfgEof: result = ""
proc getKeyValPair(c: var CfgParser, kind: CfgEventKind): CfgEvent =
if c.tok.kind == tkSymbol:
result.kind = kind

View File

@@ -10,6 +10,58 @@
## Include for the tester that contains test suites that test special features
## of the compiler.
const
specialCategories = [
"assert",
"async",
"debugger",
"dll",
"examples",
"flags",
"gc",
"io",
"js",
"lib",
"longgc",
"manyloc",
"nimble-all",
"nimble-core",
"nimble-extra",
"niminaction",
"rodfiles",
"threads",
"untestable",
"stdlib",
"testdata",
"nimcache",
"coroutines",
"osproc"
]
# these tests still have bugs. At some point when the bugs are fixd
# this should become empty.
# exclude for various reasons
const
specialDisabedTests = [
"tests/dir with space/tspace.nim", # can't import dir with spaces.
"tests/method/tmultim.nim", # (77, 8) Error: method is not a base
"tests/system/talloc2.nim", # too much memory
"tests/collections/ttables.nim", # takes too long
"tests/system/tparams.nim", # executes itself with parameters
"tests/stdlib/tquit.nim", # not testing for obvious reasons
"tests/system/trealloc.nim", # out of memory
"tests/system/t7894.nim", # causes out of memory in later tests
"tests/types/tissues_types.nim", # causes out of memory with --gc:boehm
"tests/pragmas/tused.nim", # paths in nimout differ when imported
"tests/generics/trtree.nim", # very very ugly test
"tests/array/tarray.nim", #
"tests/destructor/turn_destroy_into_finalizer.nim", # fails when imported
"tests/misc/tnew.nim",
"tests/misc/tcmdline.nim"
]
# included from tester.nim
# ---------------- ROD file tests ---------------------------------------------
@@ -327,24 +379,15 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
let filename = "tests" / test.addFileExt("nim")
let testHash = getMD5(readFile(filename).string)
doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed."
# Run the tests.
for testfile in tests:
test "tests/" & testfile & ".nim"
let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim"
testJS jsFile
let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim"
testCPP cppFile
# ------------------------- manyloc -------------------------------------------
#proc runSpecialTests(r: var TResults, options: string) =
# for t in ["lib/packages/docutils/highlite"]:
# testSpec(r, t, options)
proc findMainFile(dir: string): string =
# finds the file belonging to ".nim.cfg"; if there is no such file
@@ -382,20 +425,18 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
for testFile in os.walkFiles(pattern):
let name = extractFilename(testFile)
if name notin disabledFiles:
let contents = readFile(testFile).string
var testObj = makeTest(testFile, options, cat)
if "when isMainModule" notin contents:
testObj.spec.action = actionCompile
testSpec r, testObj
# ----------------------------- nimble ----------------------------------------
type PackageFilter = enum
pfCoreOnly
pfExtraOnly
pfAll
type
PackageFilter = enum
pfCoreOnly
pfExtraOnly
pfAll
var nimbleDir = getEnv("NIMBLE_DIR").string
if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
@@ -425,7 +466,6 @@ proc getPackageDir(package: string): string =
iterator listPackages(filter: PackageFilter): tuple[name, url: string] =
let packageList = parseFile(packageIndex)
for package in packageList.items():
let
name = package["name"].str
@@ -485,19 +525,28 @@ proc `&.?`(a, b: string): string =
# candidate for the stdlib?
result = if b.startswith(a): b else: a & b
#proc `&?.`(a, b: string): string = # not used
# candidate for the stdlib?
#result = if a.endswith(b): a else: a & b
proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
let test = "tests" & DirSep &.? cat.string / test
let target = if cat.string.normalize == "js": targetJS else: targetC
if existsFile(test):
testSpec r, makeTest(test, options, cat), {target}
else: echo "[Warning] - ", test, " test does not exist"
else:
echo "[Warning] - ", test, " test does not exist"
proc isJoinableSpec(spec: TSpec): bool
proc isJoinableSpec(spec: TSpec): bool =
result = not spec.sortoutput and
spec.action == actionRun and
spec.file.replace('\\', '/') notin specialDisabedTests and
not fileExists(spec.file.changeFileExt("cfg")) and
not fileExists(parentDir(spec.file) / "nim.cfg") and
spec.cmd.len == 0 and
spec.err != reIgnored and
spec.exitCode == 0 and
spec.input.len == 0 and
spec.nimout.len == 0 and
spec.outputCheck != ocSubstr and
spec.ccodeCheck.len == 0 and
(spec.targets == {} or spec.targets == {targetC})
proc processCategory(r: var TResults, cat: Category, options: string, runJoinableTests: bool) =
case cat.string.normalize
@@ -560,191 +609,64 @@ proc processCategory(r: var TResults, cat: Category, options: string, runJoinabl
if testsRun == 0:
echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run"
proc norm(s: var string) =
while true:
let tmp = s.replace("\n\n", "\n")
if tmp == s: break
s = tmp
s = s.strip
const specialCategories = [
"async",
"debugger",
"dll",
"examples",
"flags",
"gc",
"io",
"js",
"lib",
"longgc",
"manyloc",
"nimble-all",
"nimble-core",
"nimble-extra",
"niminaction",
"rodfiles",
"threads",
"untestable",
"stdlib",
]
# these tests still have bugs. At some point when the bugs are fixd
# this should become empty.
# exclude for various reasons
const specialDisabedTests = [
"tests/dir with space/tspace.nim", # can't import dir with spaces.
"tests/method/tmultim.nim", # (77, 8) Error: method is not a base
"tests/system/talloc2.nim", # too much memory
"tests/collections/ttables.nim", # takes too long
"tests/system/tparams.nim", # executes itself with parameters
"tests/stdlib/tquit.nim", # not testing for obvious reasons
"tests/system/trealloc.nim", # out of memory
"tests/system/t7894.nim", # causes out of memory in later tests
"tests/types/tissues_types.nim", # causes out of memory with --gc:boehm
"tests/pragmas/tused.nim", # paths in nimout differ when imported
"tests/generics/trtree.nim", # very very ugly test
"tests/array/tarray.nim", #
"tests/osproc/texecps.nim", # uses getAppFileName() to start itself with arguments
"tests/destructor/turn_destroy_into_finalizer.nim", # fails when imported
"tests/osproc/texitsignal.nim", # uses getAppFileName() to start itself with arguments
]
proc isJoinableSpec(spec: TSpec): bool =
if spec.sortoutput:
return false
if spec.action != actionRun:
return false
if spec.file in specialDisabedTests:
return false
if fileExists(spec.file & ".cfg"):
return false
if fileExists(parentDir(spec.file) / "nim.cfg"):
return false
if spec.cmd != cmdTemplate():
return false
if spec.err == reIgnored:
return false
if spec.exitCode != 0:
return false
if spec.input != "":
return false
if spec.targets != {} and spec.targets != {targetC}:
return false
return true
proc runJoinedTest(): bool =
proc runJoinedTest(testsDir: string): bool =
## returs a list of tests that have problems
var specs:seq[TSpec]
var specs: seq[TSpec] = @[]
for file in os.walkFiles("tests/*/t*.nim"):
let a = find(file, '/') + 1
let b = find(file, '/', a)
let cat = file[a ..< b]
if cat in specialCategories:
continue
let spec = parseSpec(file)
if isJoinableSpec(spec):
specs.add spec
for kind, dir in walkDir(testsDir):
assert testsDir.startsWith(testsDir)
let cat = dir[testsDir.len .. ^1]
if kind == pcDir and cat notin specialCategories:
for file in os.walkFiles(testsDir / cat / "t*.nim"):
let spec = parseSpec(file)
if isJoinableSpec(spec):
specs.add spec
echo "joinable specs: ", specs.len
var megatest: string
for runSpec in specs:
megatest.add "import \""
megatest.add "import r\""
megatest.add runSpec.file
megatest.add "\"\n"
writeFile("megatest.nim", megatest)
let args = ["c", "-d:testing", "--gc:boehm", "megatest.nim"]
const args = ["c", "-d:testing", "--listCmd", "megatest.nim"]
var (buf, exitCode) = execCmdEx2(command = "nim", args = args, options = {poStdErrToStdOut, poUsePath}, input = "")
if exitCode != 0:
echo buf
quit("megatest compilation failed")
echo "compilation ok"
var nimoutOK = true
for runSpec in specs:
for line in runSpec.nimout.splitLines:
if buf.find(line) < 0:
echo "could not find: ", line
echo runSpec.file
nimoutOK = false
if nimoutOK:
echo "nimout OK"
else:
echo "nimout FAIL"
(buf, exitCode) = execCmdEx2("./megatest", [], {poStdErrToStdOut}, "")
if exitCode != 0:
quit("megatest execution failed")
echo "run ok"
norm buf
writeFile("outputGotten.txt", buf)
var outputExpected = ""
var outputErrorCount = 0
var currentPos = 0
var lastLine = ""
# when a lot of output is skipped, this can be the cause why a later test fails.
var warnings = ""
for i, runSpec in specs:
outputExpected.add runSpec.output
if outputExpected[^1] != '\n':
outputExpected.add '\n'
outputExpected.add runSpec.output.strip
outputExpected.add '\n'
norm outputExpected
for line in runSpec.output.splitLines:
if line != "":
#if line == "2":
# echo "found the test: ", runSpec.file
let newPos = buf.find(line, currentPos)
if newPos < 0:
if outputErrorCount < 5:
echo "could not find: ", line
echo "it could be, because the test failed, or too much output is discarded by a previous search in the output."
echo warnings
warnings.setLen 0
# don't spam too much of this
if outputErrorCount == 0:
echo "############"
echo buf[currentPos-200 ..< currentPos]
echo "| (", current_pos, ")"
echo buf[currentPos ..< min(currentPos+200, buf.len)]
echo "############"
inc outputErrorCount
else:
if currentPos + lastLine.len * 2 < newPos:
warnings.addLine "Warning long skip in search for: ", line
warnings.addLine "in test: ", runSpec.file
currentPos = newPos + line.len
lastLine = line
if outputErrorCount == 0:
echo "output OK"
if buf != outputExpected:
writeFile("outputExpected.txt", outputExpected)
discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt")
echo "output different!"
result = false
else:
echo "output FAIL (", outputErrorCount, " errors)"
writeFile("outputExpected.txt", outputExpected)
# removeFile("megatest.nim")
return nimoutOK and outputErrorCount == 0
echo "output OK"
removeFile("megatest.nim")
result = true

View File

@@ -14,9 +14,6 @@ var compilerPrefix* = "compiler" / "nim"
let isTravis* = existsEnv("TRAVIS")
let isAppVeyor* = existsEnv("APPVEYOR")
proc cmdTemplate*(): string =
compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file"
type
TTestAction* = enum
actionRun = "run"
@@ -70,6 +67,12 @@ type
nimout*: string
parseErrors*: string # when the spec definition is invalid, this is not empty.
proc getCmd*(s: TSpec): string =
if s.cmd.len == 0:
result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file"
else:
result = s.cmd
const
targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"]
targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"]
@@ -97,9 +100,6 @@ proc extractSpec(filename: string): string =
when not defined(nimhygiene):
{.pragma: inject.}
proc defaultSpec*(): TSpec =
result.cmd = cmdTemplate()
proc parseTargets*(value: string): set[TTarget] =
for v in value.normalize.splitWhitespace:
case v
@@ -119,7 +119,6 @@ proc addLine*(self: var string; a,b: string) =
self.add "\n"
proc parseSpec*(filename: string): TSpec =
result = defaultSpec()
result.file = filename
let specStr = extractSpec(filename)
var ss = newStringStream(specStr)

View File

@@ -378,13 +378,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
case expected.action
of actionCompile:
var given = callCompiler(expected.cmd, test.name, test.options, target,
var given = callCompiler(expected.getCmd, test.name, test.options, target,
extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off")
compilerOutputTests(test, target, given, expected, r)
of actionRun:
# In this branch of code "early return" pattern is clearer than deep
# nested conditionals - the empty rows in between to clarify the "danger"
var given = callCompiler(expected.cmd, test.name, test.options,
var given = callCompiler(expected.getCmd, test.name, test.options,
target)
if given.err != reSuccess:
@@ -438,7 +438,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
bufB, reExitCodesDiffer)
continue
if (expected.outputCheck == ocEqual and expected.output != bufB) or
if (expected.outputCheck == ocEqual and expected.output != bufB) or
(expected.outputCheck == ocSubstr and expected.output notin bufB):
given.err = reOutputsDiffer
r.addResult(test, target, expected.output, bufB, reOutputsDiffer)
@@ -448,7 +448,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
continue
of actionReject:
var given = callCompiler(expected.cmd, test.name, test.options,
var given = callCompiler(expected.getCmd, test.name, test.options,
target)
cmpMsgs(r, expected, given, test, target)
continue
@@ -458,7 +458,7 @@ proc testC(r: var TResults, test: TTest, action: TTestAction) =
let tname = test.name.addFileExt(".c")
inc(r.total)
maybeStyledEcho "Processing ", fgCyan, extractFilename(tname)
var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC)
if given.err != reSuccess:
r.addResult(test, targetC, "", given.msg, given.err)
elif action == actionRun:
@@ -471,7 +471,7 @@ proc testExec(r: var TResults, test: TTest) =
# runs executable or script, just goes by exit code
inc(r.total)
let (outp, errC) = execCmdEx(test.options.strip())
var given: TSpec = defaultSpec()
var given: TSpec
if errC == 0:
given.err = reSuccess
else:
@@ -497,15 +497,9 @@ else:
# array of modules disabled from compilation test of stdlib.
disabledFiles = ["-"]
include categories
# proc runCaasTests(r: var TResults) =
# for test, output, status, mode in caasTestsRunner():
# r.addResult(test, "", output & "-> " & $mode,
# if status: reSuccess else: reOutputsDiffer)
const testsDir = "tests" & DirSep
proc main() =
os.putenv "NIMTEST_COLOR", "never"
@@ -556,9 +550,8 @@ proc main() =
var r = initResults()
case action
of "all":
doAssert runJoinedTest()
doAssert runJoinedTest(testsDir)
let testsDir = "tests" & DirSep
var myself = quoteShell(findExe("testament" / "tester"))
if targetsStr.len > 0:
myself &= " " & quoteShell("--targets:" & targetsStr)
@@ -587,7 +580,7 @@ proc main() =
of "html":
generateHtml(resultsFile, optFailing)
of "stats":
discard runJoinedTest()
discard runJoinedTest(testsDir)
else:
quit Usage

View File

@@ -18,5 +18,5 @@ const
(s: "hi", x: 69, y: 45, z: 0.0, chars: {'a', 'b', 'c'}),
(s: "hi", x: 69, y: 45, z: 1.0, chars: {'a'})]
write(stdout, things[0].x)
writeLine(stdout, things[0].x)
#OUT 69

View File

@@ -85,13 +85,13 @@ block tnestif:
if x == 0:
write(stdout, "i == 0")
if y == 0:
write(stdout, x)
writeLine(stdout, x)
else:
write(stdout, y)
writeLine(stdout, y)
elif x == 1:
write(stdout, "i == 1")
writeLine(stdout, "i == 1")
elif x == 2:
write(stdout, "i == 2")
writeLine(stdout, "i == 2")
else:
write(stdout, "looks like Python")
writeLine(stdout, "looks like Python")
#OUT i == 2

View File

@@ -1,14 +1,15 @@
discard """
output: "Nim 3.4368930843, 0.3299290698 C double: 3.4368930843, 0.3299290698"
output: '''
Nim 3.4368930843, 0.3299290698
C double: 3.4368930843, 0.3299290698'''
"""
import math, strutils
{.emit: """
void printFloats(void) {
double y = 1.234567890123456789;
printf("C double: %.10f, %.10f ", exp(y), cos(y));
double y = 1.234567890123456789;
printf("C double: %.10f, %.10f\n", exp(y), cos(y));
}
""".}
@@ -16,5 +17,5 @@ proc c_printf(frmt: cstring) {.importc: "printf", header: "<stdio.h>", varargs.}
proc printFloats {.importc, nodecl.}
var x: float = 1.234567890123456789
c_printf("Nim %.10f, %.10f ", exp(x), cos(x))
c_printf("Nim %.10f, %.10f\n", exp(x), cos(x))
printFloats()

View File

@@ -25,3 +25,4 @@ for y in items(r2):
stdout.write(y)
#OUT helloworld99110223
stdout.write "\n"

View File

@@ -101,4 +101,6 @@ when true:
for y in items(r2):
stdout.write(y)
stdout.write "\n"
#OUT helloworld99110223

View File

@@ -32,3 +32,5 @@ c.add('A', "12")
c.add('B', "13")
for k, v in items(c):
stdout.write(" Key: ", $k, " value: ", v)
stdout.write "\n"

View File

@@ -33,3 +33,4 @@ c.add('B', "13")
for k, v in items(c):
stdout.write(" Key: ", $k, " value: ", v)
stdout.write "\n"

View File

@@ -1,6 +1,7 @@
discard """
output: '''
100 0
100
0
'''
"""
@@ -72,7 +73,7 @@ block tgeneric1:
proc print[T](heap: PBinHeap[T]) =
for i in countup(0, heap.last):
stdout.write($heap.heap[i].data, " ")
stdout.write($heap.heap[i].data, "\n")
var heap: PBinHeap[int]

View File

@@ -34,3 +34,4 @@ proc test(data: seq[int], value: int): seq[int] =
for x in items(test(@[1,2,3], 2)):
stdout.write(x)
stdout.write "\n"

View File

@@ -2,9 +2,8 @@ discard """
output: "0123456789"
"""
# Test new countup and unary <
# Test new countup
for i in 0 .. < 10'i64:
for i in 0 ..< 10'i64:
stdout.write(i)
#OUT 0123456789
echo "\n"

View File

@@ -10,3 +10,4 @@ for x in lines.split():
stdout.write(x)
#OUT abcxyz
stdout.write "\n"

View File

@@ -31,3 +31,5 @@ proc wordWrap2(s: string, maxLineWidth = 80,
result = ""
for word, isSep in tokenize2(s, seps):
var w = 0
stdout.write "\n"

View File

@@ -45,9 +45,9 @@ proc square(x:int): int =
let list = @[1,2,3,4,5,6,7,8,9]
echo ("--- evens")
for item in list / isEven : echo(item)
for item in list / isEven: echo(item)
echo ("--- squares")
for item in list >> square : echo(item)
for item in list >> square: echo(item)
#echo ("--- squares of evens, only")
# next line doesn't compile. Generic types are not inferred
#for item in list />> (isEven, square) : echo(item)

View File

@@ -10,3 +10,5 @@ iterator iterAndZero(a: var openArray[int]): int =
var x = [[1, 2, 3], [4, 5, 6]]
for y in iterAndZero(x[0]): write(stdout, $y)
#OUT 123
write stdout, "\n"

View File

@@ -70,3 +70,4 @@ var
s2 = formatStyleInterpolation"Hello ${bob}, ${sum(alice.len, bob.len, 2)}$$"
write(stdout, s1 & " | " & s2)
write(stdout, "\n")

View File

@@ -8,10 +8,10 @@ type
TMatrix2x2*[T] = TMatrixNM[range[0..1], range[0..1], T]
TMatrix3x3*[T] = TMatrixNM[range[0..2], range[0..2], T]
proc test*[T] (matrix: TMatrix2x2[T]) =
proc test*[T](matrix: TMatrix2x2[T]) =
echo "wrong proc called"
proc test*[T] (matrix: TMatrix3x3[T]) =
proc test*[T](matrix: TMatrix3x3[T]) =
echo "right proc called"
var matrix: TMatrix3x3[float]

View File

@@ -43,3 +43,4 @@ for i in 0..w-1:
for i in 0..w-1:
stdout.write(m[i,i]) #OUT 111
stdout.write "\n"

View File

@@ -16,3 +16,4 @@ proc ack(x, y: int): int =
# echo(ack(0, 0))
write(stdout, ack(3, 4)) #OUT 125
write stdout, "\n"

View File

@@ -29,4 +29,5 @@ proc mypos(sub, s: string, start: int = 0): int =
var sub = "hello"
var s = "world hello"
write(stdout, mypos(sub, s))
write stdout, "\n"
#OUT 6

View File

@@ -203,3 +203,4 @@ block ttoseq:
stdout.write(x)
var y: type("a b c".split)
y = "xzy"
stdout.write("\n")

View File

@@ -65,7 +65,7 @@ proc reindent*(value: string, preset_indent = 0): string =
make([ html, xml, glsl, js, css, rst ])
when true:
when isMainModule:
## Test tags
const script = js"""

View File

@@ -84,3 +84,5 @@ proc use(r: var TResource) =
autoClose(r = openResource("test")):
use r
write stdout, "\n"

View File

@@ -9,4 +9,4 @@ var
a, b: int = 4
write(stdout, a)
write(stdout, b) #OUT 44
writeLine(stdout, b) #OUT 44

View File

@@ -70,8 +70,9 @@ block:
# Tests for VM ops
block:
static:
# for joint test, the project path is different
assert "vm" in getProjectPath() or "Nim" in getProjectPath()
# for joint test, the project path is different, so I disabled it:
when false:
assert "vm" in getProjectPath()
let b = getEnv("UNSETENVVAR")
assert b == ""