mirror of
https://github.com/nim-lang/Nim.git
synced 2025-12-30 18:02:05 +00:00
414 lines
13 KiB
Nim
Executable File
414 lines
13 KiB
Nim
Executable File
#
|
|
#
|
|
# Nimrod Tester
|
|
# (c) Copyright 2012 Andreas Rumpf
|
|
#
|
|
# See the file "copying.txt", included in this
|
|
# distribution, for details about the copyright.
|
|
#
|
|
|
|
## This program verifies Nimrod against the testcases.
|
|
|
|
import
|
|
parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json,
|
|
marshal, cgi, parseopt
|
|
|
|
const
|
|
cmdTemplate = r"nimrod cc --hints:on $# $#"
|
|
resultsFile = "testresults.html"
|
|
jsonFile = "testresults.json"
|
|
Usage = "usage: tester [--print] " &
|
|
"reject|compile|run|" &
|
|
"merge|special|rodfiles| [nimrod options]\n" &
|
|
" or: tester test|comp|rej singleTest"
|
|
|
|
type
|
|
TTestAction = enum
|
|
actionCompile, actionRun, actionReject
|
|
TSpec {.pure.} = object
|
|
action: TTestAction
|
|
file, cmd: string
|
|
outp: string
|
|
line, exitCode: int
|
|
msg: string
|
|
err: bool
|
|
disabled: bool
|
|
substr: bool
|
|
TResults {.pure.} = object
|
|
total, passed, skipped: int
|
|
data: string
|
|
|
|
TResultEnum = enum
|
|
reFailure, reIgnored, reSuccess
|
|
|
|
# ----------------------- Spec parser ----------------------------------------
|
|
|
|
when not defined(parseCfgBool):
|
|
# candidate for the stdlib:
|
|
proc parseCfgBool(s: string): bool =
|
|
case normalize(s)
|
|
of "y", "yes", "true", "1", "on": result = true
|
|
of "n", "no", "false", "0", "off": result = false
|
|
else: raise newException(EInvalidValue, "cannot interpret as a bool: " & s)
|
|
|
|
proc extractSpec(filename: string): string =
|
|
const tripleQuote = "\"\"\""
|
|
var x = readFile(filename).string
|
|
var a = x.find(tripleQuote)
|
|
var b = x.find(tripleQuote, a+3)
|
|
# look for """ only in the first section
|
|
if a >= 0 and b > a and a < 40:
|
|
result = x.substr(a+3, b-1).replace("'''", tripleQuote)
|
|
else:
|
|
#echo "warning: file does not contain spec: " & filename
|
|
result = ""
|
|
|
|
when not defined(nimhygiene):
|
|
{.pragma: inject.}
|
|
|
|
template parseSpecAux(fillResult: stmt) {.immediate.} =
|
|
var ss = newStringStream(extractSpec(filename))
|
|
var p {.inject.}: TCfgParser
|
|
open(p, ss, filename, 1)
|
|
while true:
|
|
var e {.inject.} = next(p)
|
|
case e.kind
|
|
of cfgEof: break
|
|
of cfgSectionStart, cfgOption, cfgError:
|
|
echo ignoreMsg(p, e)
|
|
of cfgKeyValuePair:
|
|
fillResult
|
|
close(p)
|
|
|
|
proc parseSpec(filename: string): TSpec =
|
|
result.file = filename
|
|
result.err = true
|
|
result.msg = ""
|
|
result.outp = ""
|
|
result.cmd = cmdTemplate
|
|
parseSpecAux:
|
|
case normalize(e.key)
|
|
of "action":
|
|
case e.value.normalize
|
|
of "compile": result.action = actionCompile
|
|
of "run": result.action = actionRun
|
|
of "reject": result.action = actionReject
|
|
else: echo ignoreMsg(p, e)
|
|
of "file": result.file = e.value
|
|
of "line": discard parseInt(e.value, result.line)
|
|
of "output": result.outp = e.value
|
|
of "outputsub":
|
|
result.outp = e.value
|
|
result.substr = true
|
|
of "exitcode":
|
|
discard parseInt(e.value, result.exitCode)
|
|
of "errormsg", "msg": result.msg = e.value
|
|
of "disabled": result.disabled = parseCfgBool(e.value)
|
|
of "cmd": result.cmd = e.value
|
|
else: echo ignoreMsg(p, e)
|
|
|
|
# ----------------------------------------------------------------------------
|
|
|
|
let
|
|
pegLineError =
|
|
peg"{[^(]*} '(' {\d+} ', ' \d+ ') ' ('Error'/'Warning') ':' \s* {.*}"
|
|
pegOtherError = peg"'Error:' \s* {.*}"
|
|
pegSuccess = peg"'Hint: operation successful'.*"
|
|
pegOfInterest = pegLineError / pegOtherError
|
|
|
|
proc callCompiler(cmdTemplate, filename, options: string): TSpec =
|
|
let c = parseCmdLine(cmdTemplate % [options, filename])
|
|
var p = startProcess(command=c[0], args=c[1.. -1],
|
|
options={poStdErrToStdOut, poUseShell})
|
|
let outp = p.outputStream
|
|
var suc = ""
|
|
var err = ""
|
|
var x = newStringOfCap(120)
|
|
while outp.readLine(x.TaintedString) or running(p):
|
|
if x =~ pegOfInterest:
|
|
# `err` should contain the last error/warning message
|
|
err = x
|
|
elif x =~ pegSuccess:
|
|
suc = x
|
|
close(p)
|
|
result.msg = ""
|
|
result.file = ""
|
|
result.outp = ""
|
|
result.err = true
|
|
result.line = -1
|
|
if err =~ pegLineError:
|
|
result.file = extractFilename(matches[0])
|
|
result.line = parseInt(matches[1])
|
|
result.msg = matches[2]
|
|
elif err =~ pegOtherError:
|
|
result.msg = matches[0]
|
|
elif suc =~ pegSuccess:
|
|
result.err = false
|
|
|
|
proc initResults: TResults =
|
|
result.total = 0
|
|
result.passed = 0
|
|
result.skipped = 0
|
|
result.data = ""
|
|
|
|
proc readResults(filename: string): TResults =
|
|
result = marshal.to[TResults](readFile(filename).string)
|
|
|
|
proc writeResults(filename: string, r: TResults) =
|
|
writeFile(filename, $$r)
|
|
|
|
proc `$`(x: TResults): string =
|
|
result = ("Tests passed: $1 / $3 <br />\n" &
|
|
"Tests skipped: $2 / $3 <br />\n") %
|
|
[$x.passed, $x.skipped, $x.total]
|
|
|
|
proc colorResult(r: TResultEnum): string =
|
|
case r
|
|
of reFailure: result = "<span style=\"color:red\">no</span>"
|
|
of reIgnored: result = "<span style=\"color:fuchsia\">ignored</span>"
|
|
of reSuccess: result = "<span style=\"color:green\">yes</span>"
|
|
|
|
const
|
|
TableHeader4 = "<table border=\"1\"><tr><td>Test</td><td>Expected</td>" &
|
|
"<td>Given</td><td>Success</td></tr>\n"
|
|
TableHeader3 = "<table border=\"1\"><tr><td>Test</td>" &
|
|
"<td>Given</td><td>Success</td></tr>\n"
|
|
TableFooter = "</table>\n"
|
|
HtmlBegin = """<html>
|
|
<head>
|
|
<title>Test results</title>
|
|
<style type="text/css">
|
|
<!--""" & slurp("css/boilerplate.css") & "\n" &
|
|
slurp("css/style.css") &
|
|
"""-->
|
|
</style>
|
|
|
|
</head>
|
|
<body>"""
|
|
|
|
HtmlEnd = "</body></html>"
|
|
|
|
proc td(s: string): string =
|
|
result = s.substr(0, 200).XMLEncode
|
|
|
|
proc addResult(r: var TResults, test, expected, given: string,
|
|
success: TResultEnum) =
|
|
r.data.addf("<tr><td>$#</td><td>$#</td><td>$#</td><td>$#</td></tr>\n", [
|
|
XMLEncode(test), td(expected), td(given), success.colorResult])
|
|
|
|
proc addResult(r: var TResults, test, given: string,
|
|
success: TResultEnum) =
|
|
r.data.addf("<tr><td>$#</td><td>$#</td><td>$#</td></tr>\n", [
|
|
XMLEncode(test), td(given), success.colorResult])
|
|
|
|
proc listResults(reject, compile, run: TResults) =
|
|
var s = HtmlBegin
|
|
s.add("<h1>Tests to Reject</h1>\n")
|
|
s.add($reject)
|
|
s.add(TableHeader4 & reject.data & TableFooter)
|
|
s.add("<br /><br /><br /><h1>Tests to Compile</h1>\n")
|
|
s.add($compile)
|
|
s.add(TableHeader3 & compile.data & TableFooter)
|
|
s.add("<br /><br /><br /><h1>Tests to Run</h1>\n")
|
|
s.add($run)
|
|
s.add(TableHeader4 & run.data & TableFooter)
|
|
s.add(HtmlEnd)
|
|
writeFile(resultsFile, s)
|
|
|
|
proc cmpMsgs(r: var TResults, expected, given: TSpec, test: string) =
|
|
if strip(expected.msg) notin strip(given.msg):
|
|
r.addResult(test, expected.msg, given.msg, reFailure)
|
|
elif extractFilename(expected.file) != extractFilename(given.file) and
|
|
"internal error:" notin expected.msg:
|
|
r.addResult(test, expected.file, given.file, reFailure)
|
|
elif expected.line != given.line and expected.line != 0:
|
|
r.addResult(test, $expected.line, $given.line, reFailure)
|
|
else:
|
|
r.addResult(test, expected.msg, given.msg, reSuccess)
|
|
inc(r.passed)
|
|
|
|
proc rejectSingleTest(r: var TResults, test, options: string) =
|
|
let test = test.addFileExt(".nim")
|
|
var t = extractFilename(test)
|
|
inc(r.total)
|
|
echo t
|
|
var expected = parseSpec(test)
|
|
if expected.disabled:
|
|
r.addResult(t, "", "", reIgnored)
|
|
inc(r.skipped)
|
|
else:
|
|
var given = callCompiler(expected.cmd, test, options)
|
|
cmpMsgs(r, expected, given, t)
|
|
|
|
proc reject(r: var TResults, dir, options: string) =
|
|
## handle all the tests that the compiler should reject
|
|
for test in os.walkFiles(dir / "t*.nim"): rejectSingleTest(r, test, options)
|
|
|
|
proc compile(r: var TResults, pattern, options: string) =
|
|
for test in os.walkFiles(pattern):
|
|
let t = extractFilename(test)
|
|
echo t
|
|
inc(r.total)
|
|
let expected = parseSpec(test)
|
|
if expected.disabled:
|
|
r.addResult(t, "", reIgnored)
|
|
inc(r.skipped)
|
|
else:
|
|
var given = callCompiler(expected.cmd, test, options)
|
|
r.addResult(t, given.msg, if given.err: reFailure else: reSuccess)
|
|
if not given.err: inc(r.passed)
|
|
|
|
proc compileSingleTest(r: var TResults, test, options: string) =
|
|
let test = test.addFileExt(".nim")
|
|
let t = extractFilename(test)
|
|
inc(r.total)
|
|
echo t
|
|
let given = callCompiler(cmdTemplate, test, options)
|
|
r.addResult(t, given.msg, if given.err: reFailure else: reSuccess)
|
|
if not given.err: inc(r.passed)
|
|
|
|
type
|
|
TTarget = enum targetC, targetJS
|
|
|
|
proc runSingleTest(r: var TResults, test, options: string, target: TTarget) =
|
|
var test = test.addFileExt(".nim")
|
|
var t = extractFilename(test)
|
|
echo t
|
|
inc(r.total)
|
|
var expected = parseSpec(test)
|
|
if expected.disabled:
|
|
r.addResult(t, "", "", reIgnored)
|
|
inc(r.skipped)
|
|
else:
|
|
var given = callCompiler(expected.cmd, test, options)
|
|
if given.err:
|
|
r.addResult(t, "", given.msg, reFailure)
|
|
else:
|
|
var exeFile: string
|
|
if target == targetC:
|
|
exeFile = changeFileExt(test, ExeExt)
|
|
else:
|
|
let (dir, file, ext) = splitFile(test)
|
|
exeFile = dir / "nimcache" / file & ".js"
|
|
|
|
if existsFile(exeFile):
|
|
var (buf, exitCode) = execCmdEx(
|
|
(if target==targetJS: "node " else: "") & exeFile)
|
|
if exitCode != expected.ExitCode:
|
|
r.addResult(t, "exitcode: " & $expected.ExitCode,
|
|
"exitcode: " & $exitCode, reFailure)
|
|
else:
|
|
var success = strip(buf.string) == strip(expected.outp)
|
|
if expected.substr and not success:
|
|
success = expected.outp in buf.string
|
|
if success: inc(r.passed)
|
|
r.addResult(t, expected.outp,
|
|
buf.string, if success: reSuccess else: reFailure)
|
|
else:
|
|
r.addResult(t, expected.outp, "executable not found", reFailure)
|
|
|
|
proc runSingleTest(r: var TResults, test, options: string) =
|
|
runSingleTest(r, test, options, targetC)
|
|
|
|
proc run(r: var TResults, dir, options: string) =
|
|
for test in os.walkFiles(dir / "t*.nim"): runSingleTest(r, test, options)
|
|
|
|
include specials
|
|
|
|
proc compileExample(r: var TResults, pattern, options: string) =
|
|
for test in os.walkFiles(pattern): compileSingleTest(r, test, options)
|
|
|
|
proc toJson(res: TResults): PJsonNode =
|
|
result = newJObject()
|
|
result["total"] = newJInt(res.total)
|
|
result["passed"] = newJInt(res.passed)
|
|
result["skipped"] = newJInt(res.skipped)
|
|
|
|
proc outputJSON(reject, compile, run: TResults) =
|
|
var doc = newJObject()
|
|
doc["reject"] = toJson(reject)
|
|
doc["compile"] = toJson(compile)
|
|
doc["run"] = toJson(run)
|
|
var s = pretty(doc)
|
|
writeFile(jsonFile, s)
|
|
|
|
proc main() =
|
|
os.putenv "NIMTEST_NO_COLOR", "1"
|
|
os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
|
|
|
|
const
|
|
compileJson = "compile.json"
|
|
runJson = "run.json"
|
|
rejectJson = "reject.json"
|
|
|
|
var optPrintResults = false
|
|
var p = initOptParser()
|
|
p.next()
|
|
if p.kind == cmdLongoption:
|
|
case p.key.string
|
|
of "print": optPrintResults = true
|
|
else: quit usage
|
|
p.next()
|
|
if p.kind != cmdArgument: quit usage
|
|
var action = p.key.string.normalize
|
|
p.next()
|
|
var r = initResults()
|
|
case action
|
|
of "reject":
|
|
reject(r, "tests/reject", p.cmdLineRest.string)
|
|
rejectSpecialTests(r, p.cmdLineRest.string)
|
|
writeResults(rejectJson, r)
|
|
of "compile":
|
|
compile(r, "tests/compile/t*.nim", p.cmdLineRest.string)
|
|
compile(r, "tests/ccg/t*.nim", p.cmdLineRest.string)
|
|
compile(r, "tests/ecmas.nim", p.cmdLineRest.string)
|
|
compileExample(r, "lib/pure/*.nim", p.cmdLineRest.string)
|
|
compileExample(r, "examples/*.nim", p.cmdLineRest.string)
|
|
compileExample(r, "examples/gtk/*.nim", p.cmdLineRest.string)
|
|
compileSpecialTests(r, p.cmdLineRest.string)
|
|
writeResults(compileJson, r)
|
|
of "run":
|
|
run(r, "tests/run", p.cmdLineRest.string)
|
|
runSpecialTests(r, p.cmdLineRest.string)
|
|
writeResults(runJson, r)
|
|
of "special":
|
|
runSpecialTests(r, p.cmdLineRest.string)
|
|
writeResults(runJson, r)
|
|
of "rodfiles":
|
|
runRodFiles(r, p.cmdLineRest.string)
|
|
writeResults(runJson, r)
|
|
of "js":
|
|
if existsFile(runJSon):
|
|
r = readResults(runJson)
|
|
runJsTests(r, p.cmdLineRest.string)
|
|
writeResults(runJson, r)
|
|
of "merge":
|
|
var rejectRes = readResults(rejectJson)
|
|
var compileRes = readResults(compileJson)
|
|
var runRes = readResults(runJson)
|
|
listResults(rejectRes, compileRes, runRes)
|
|
outputJSON(rejectRes, compileRes, runRes)
|
|
of "dll":
|
|
runDLLTests r, p.cmdLineRest.string
|
|
of "gc":
|
|
runGCTests(r, p.cmdLineRest.string)
|
|
of "test", "comp", "rej":
|
|
if p.kind != cmdArgument: quit usage
|
|
var testFile = p.key.string
|
|
p.next()
|
|
if peg"'/reject/'" in testFile or action == "rej":
|
|
rejectSingleTest(r, testFile, p.cmdLineRest.string)
|
|
elif peg"'/compile/'" in testFile or action == "comp":
|
|
compileSingleTest(r, testFile, p.cmdLineRest.string)
|
|
else:
|
|
runSingleTest(r, testFile, p.cmdLineRest.string)
|
|
else:
|
|
quit usage
|
|
|
|
if optPrintResults: echo r, r.data
|
|
|
|
if paramCount() == 0:
|
|
quit usage
|
|
main()
|
|
|