mirror of
https://github.com/nim-lang/Nim.git
synced 2026-06-03 02:18:00 +00:00
new tester produces json output
This commit is contained in:
Araq
5
koch.nim
5
koch.nim
@@ -266,8 +266,9 @@ proc tests(args: string) =
|
||||
# we compile the tester with taintMode:on to have a basic
|
||||
# taint mode test :-)
|
||||
exec "nimrod cc --taintMode:on tests/testament/tester"
|
||||
exec quoteShell(getCurrentDir() / "tests/testament/tester".exe) & " " &
|
||||
(args|"all")
|
||||
let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe)
|
||||
exec tester & " " & (args|"all")
|
||||
exec tester & " html"
|
||||
|
||||
proc temp(args: string) =
|
||||
var output = "compiler" / "nimrod".exe
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
#
|
||||
#
|
||||
# Nimrod Tester
|
||||
# (c) Copyright 2013 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
## Include for the tester that contains test suites that test special features
|
||||
## of the compiler.
|
||||
|
||||
# included from tester.nim
|
||||
# ---------------- ROD file tests ---------------------------------------------
|
||||
|
||||
const
|
||||
rodfilesDir = "tests/rodfiles"
|
||||
nimcacheDir = rodfilesDir / "nimcache"
|
||||
|
||||
proc delNimCache() =
|
||||
try:
|
||||
removeDir(nimcacheDir)
|
||||
except EOS:
|
||||
echo "[Warning] could not delete: ", nimcacheDir
|
||||
|
||||
proc runRodFiles(r: var TResults, options: string) =
|
||||
template test(filename: expr): stmt =
|
||||
runSingleTest(r, rodfilesDir / filename, options)
|
||||
|
||||
delNimCache()
|
||||
|
||||
# test basic recompilation scheme:
|
||||
test "hallo"
|
||||
test "hallo"
|
||||
# test incremental type information:
|
||||
test "hallo2"
|
||||
delNimCache()
|
||||
|
||||
# test type converters:
|
||||
test "aconv"
|
||||
test "bconv"
|
||||
delNimCache()
|
||||
|
||||
# test G, A, B example from the documentation; test init sections:
|
||||
test "deada"
|
||||
test "deada2"
|
||||
delNimCache()
|
||||
|
||||
# test method generation:
|
||||
test "bmethods"
|
||||
test "bmethods2"
|
||||
delNimCache()
|
||||
|
||||
# test generics:
|
||||
test "tgeneric1"
|
||||
test "tgeneric2"
|
||||
delNimCache()
|
||||
|
||||
proc compileRodFiles(r: var TResults, options: string) =
|
||||
template test(filename: expr): stmt =
|
||||
compileSingleTest(r, rodfilesDir / filename, options)
|
||||
|
||||
delNimCache()
|
||||
# test DLL interfacing:
|
||||
test "gtkex1"
|
||||
test "gtkex2"
|
||||
delNimCache()
|
||||
|
||||
# --------------------- DLL generation tests ----------------------------------
|
||||
|
||||
proc safeCopyFile(src, dest: string) =
|
||||
try:
|
||||
copyFile(src, dest)
|
||||
except EOS:
|
||||
echo "[Warning] could not copy: ", src, " to ", dest
|
||||
|
||||
proc runBasicDLLTest(c, r: var TResults, options: string) =
|
||||
compileSingleTest c, "lib/nimrtl.nim", options & " --app:lib -d:createNimRtl"
|
||||
compileSingleTest c, "tests/dll/server.nim",
|
||||
options & " --app:lib -d:useNimRtl"
|
||||
|
||||
when defined(Windows):
|
||||
# windows looks in the dir of the exe (yay!):
|
||||
var nimrtlDll = DynlibFormat % "nimrtl"
|
||||
safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
|
||||
else:
|
||||
# posix relies on crappy LD_LIBRARY_PATH (ugh!):
|
||||
var libpath = getenv"LD_LIBRARY_PATH".string
|
||||
if peg"\i '/nimrod' (!'/')* '/lib'" notin libpath:
|
||||
echo "[Warning] insufficient LD_LIBRARY_PATH"
|
||||
var serverDll = DynlibFormat % "server"
|
||||
safeCopyFile("tests/dll" / serverDll, "lib" / serverDll)
|
||||
|
||||
runSingleTest r, "tests/dll/client.nim", options & " -d:useNimRtl"
|
||||
|
||||
proc runDLLTests(r: var TResults, options: string) =
|
||||
# dummy compile result:
|
||||
var c = initResults()
|
||||
|
||||
runBasicDLLTest c, r, options
|
||||
runBasicDLLTest c, r, options & " -d:release"
|
||||
runBasicDLLTest c, r, options & " --gc:boehm"
|
||||
runBasicDLLTest c, r, options & " -d:release --gc:boehm"
|
||||
|
||||
proc compileDLLTests(r: var TResults, options: string) =
|
||||
# dummy run result:
|
||||
var c = initResults()
|
||||
|
||||
runBasicDLLTest r, c, options
|
||||
runBasicDLLTest r, c, options & " -d:release"
|
||||
runBasicDLLTest r, c, options & " --gc:boehm"
|
||||
runBasicDLLTest r, c, options & " -d:release --gc:boehm"
|
||||
|
||||
# ------------------------------ GC tests -------------------------------------
|
||||
|
||||
proc runGcTests(r: var TResults, options: string) =
|
||||
template test(filename: expr): stmt =
|
||||
runSingleTest(r, "tests/gc" / filename, options)
|
||||
runSingleTest(r, "tests/gc" / filename, options & " -d:release")
|
||||
runSingleTest(r, "tests/gc" / filename, options &
|
||||
" -d:release -d:useRealtimeGC")
|
||||
runSingleTest(r, "tests/gc" / filename, options &
|
||||
" --gc:markAndSweep")
|
||||
runSingleTest(r, "tests/gc" / filename, options &
|
||||
" -d:release --gc:markAndSweep")
|
||||
|
||||
test "gcbench"
|
||||
test "gcleak"
|
||||
test "gcleak2"
|
||||
test "gctest"
|
||||
test "gcleak3"
|
||||
test "weakrefs"
|
||||
test "cycleleak"
|
||||
test "closureleak"
|
||||
|
||||
# ------------------------- threading tests -----------------------------------
|
||||
|
||||
proc runThreadTests(r: var TResults, options: string) =
|
||||
template test(filename: expr): stmt =
|
||||
runSingleTest(r, "tests/threads" / filename, options)
|
||||
runSingleTest(r, "tests/threads" / filename, options & " -d:release")
|
||||
runSingleTest(r, "tests/threads" / filename, options & " --tlsEmulation:on")
|
||||
|
||||
test "tactors"
|
||||
test "tactors2"
|
||||
test "threadex"
|
||||
# deactivated because output capturing still causes problems sometimes:
|
||||
#test "trecursive_actor"
|
||||
#test "threadring"
|
||||
#test "tthreadanalysis"
|
||||
#test "tthreadsort"
|
||||
|
||||
proc rejectThreadTests(r: var TResults, options: string) =
|
||||
rejectSingleTest(r, "tests/threads/tthreadanalysis2", options)
|
||||
rejectSingleTest(r, "tests/threads/tthreadanalysis3", options)
|
||||
rejectSingleTest(r, "tests/threads/tthreadheapviolation1", options)
|
||||
|
||||
# ------------------------- IO tests ------------------------------------------
|
||||
|
||||
proc runIOTests(r: var TResults, options: string) =
|
||||
# We need readall_echo to be compiled for this test to run.
|
||||
# dummy compile result:
|
||||
var c = initResults()
|
||||
compileSingleTest(c, "tests/system/helpers/readall_echo", options)
|
||||
runSingleTest(r, "tests/system/io", options)
|
||||
|
||||
# ------------------------- debugger tests ------------------------------------
|
||||
|
||||
proc compileDebuggerTests(r: var TResults, options: string) =
|
||||
compileSingleTest(r, "tools/nimgrep", options &
|
||||
" --debugger:on")
|
||||
|
||||
# ------------------------- JS tests ------------------------------------------
|
||||
|
||||
proc runJsTests(r: var TResults, options: string) =
|
||||
template test(filename: expr): stmt =
|
||||
runSingleTest(r, filename, options & " -d:nodejs", targetJS)
|
||||
runSingleTest(r, filename, options & " -d:nodejs -d:release", targetJS)
|
||||
|
||||
for t in os.walkFiles("tests/js/t*.nim"):
|
||||
test(t)
|
||||
for testfile in ["texceptions", "texcpt1", "texcsub", "tfinally",
|
||||
"tfinally2", "tfinally3", "tactiontable", "tmultim1",
|
||||
"tmultim3", "tmultim4"]:
|
||||
test "tests/run/" & testfile & ".nim"
|
||||
|
||||
# ------------------------- register special tests here -----------------------
|
||||
proc runSpecialTests(r: var TResults, options: string) =
|
||||
runRodFiles(r, options)
|
||||
#runDLLTests(r, options)
|
||||
runGCTests(r, options)
|
||||
runThreadTests(r, options & " --threads:on")
|
||||
runIOTests(r, options)
|
||||
|
||||
for t in os.walkFiles("tests/patterns/t*.nim"):
|
||||
runSingleTest(r, t, options)
|
||||
for t in ["lib/packages/docutils/highlite"]:
|
||||
runSingleTest(r, t, options)
|
||||
|
||||
proc rejectSpecialTests(r: var TResults, options: string) =
|
||||
rejectThreadTests(r, options)
|
||||
|
||||
proc findMainFile(dir: string): string =
|
||||
# finds the file belonging to ".nimrod.cfg"; if there is no such file
|
||||
# it returns the some ".nim" file if there is only one:
|
||||
const cfgExt = ".nimrod.cfg"
|
||||
result = ""
|
||||
var nimFiles = 0
|
||||
for kind, file in os.walkDir(dir):
|
||||
if kind == pcFile:
|
||||
if file.endsWith(cfgExt): return file[.. -(cfgExt.len+1)] & ".nim"
|
||||
elif file.endsWith(".nim"):
|
||||
if result.len == 0: result = file
|
||||
inc nimFiles
|
||||
if nimFiles != 1: result.setlen(0)
|
||||
|
||||
proc compileManyLoc(r: var TResults, options: string) =
|
||||
for kind, dir in os.walkDir("tests/manyloc"):
|
||||
if kind == pcDir:
|
||||
let mainfile = findMainFile(dir)
|
||||
if mainfile != ".nim":
|
||||
compileSingleTest(r, mainfile, options)
|
||||
|
||||
proc compileSpecialTests(r: var TResults, options: string) =
|
||||
compileRodFiles(r, options)
|
||||
|
||||
compileSingleTest(r, "compiler/c2nim/c2nim.nim", options)
|
||||
compileSingleTest(r, "compiler/pas2nim/pas2nim.nim", options)
|
||||
|
||||
compileDLLTests(r, options)
|
||||
compileDebuggerTests(r, options)
|
||||
|
||||
compileManyLoc(r, options)
|
||||
|
||||
#var given = callCompiler("nimrod i", "nimrod i", options)
|
||||
#r.addResult("nimrod i", given.msg, if given.err: reFailure else: reSuccess)
|
||||
#if not given.err: inc(r.passed)
|
||||
|
||||
@@ -49,10 +49,10 @@ proc createDb() =
|
||||
# """, [])
|
||||
|
||||
type
|
||||
MachineId = distinct int64
|
||||
MachineId* = distinct int64
|
||||
CommitId = distinct int64
|
||||
|
||||
proc `$`(id: MachineId): string {.borrow.}
|
||||
proc `$`*(id: MachineId): string {.borrow.}
|
||||
proc `$`(id: CommitId): string {.borrow.}
|
||||
|
||||
var
|
||||
@@ -61,7 +61,7 @@ var
|
||||
|
||||
proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip
|
||||
|
||||
proc getMachine: MachineId =
|
||||
proc getMachine*(db: TDbConn): MachineId =
|
||||
var name = "hostname"()
|
||||
if name.len == 0:
|
||||
name = when defined(posix): getenv"HOSTNAME".string
|
||||
@@ -76,7 +76,7 @@ proc getMachine: MachineId =
|
||||
result = db.insertId(sql"insert into Machine(name, os, cpu) values (?,?,?)",
|
||||
name, system.hostOS, system.hostCPU).MachineId
|
||||
|
||||
proc getCommit: CommitId =
|
||||
proc getCommit(db: TDbConn): CommitId =
|
||||
const commLen = "commit ".len
|
||||
let hash = "git log -n 1"()[commLen..commLen+10]
|
||||
let branch = "git symbolic-ref --short HEAD"()
|
||||
@@ -115,7 +115,7 @@ proc open*() =
|
||||
db = open(connection="testament.db", user="testament", password="",
|
||||
database="testament")
|
||||
createDb()
|
||||
thisMachine = getMachine()
|
||||
thisCommit = getCommit()
|
||||
thisMachine = getMachine(db)
|
||||
thisCommit = getCommit(db)
|
||||
|
||||
proc close*() = close(db)
|
||||
|
||||
@@ -159,3 +159,22 @@ proc generateHtml*(filename: string, commit: int) =
|
||||
outfile.write(HtmlEnd)
|
||||
close(db)
|
||||
close(outfile)
|
||||
|
||||
proc generateJson*(filename: string, commit: int) =
|
||||
const selRow = """select count(*),
|
||||
sum(result = 'reSuccess'),
|
||||
sum(result = 'reIgnored')
|
||||
from TestResult
|
||||
where [commit] = ? and machine = ?
|
||||
order by category"""
|
||||
var db = open(connection="testament.db", user="testament", password="",
|
||||
database="testament")
|
||||
let lastCommit = db.getCommit(commit)
|
||||
|
||||
var outfile = open(filename, fmWrite)
|
||||
|
||||
let data = db.getRow(sql(selRow), lastCommit, $backend.getMachine(db))
|
||||
|
||||
outfile.writeln("""{"total": $#, "passed": $#, "skipped": $#}""" % data)
|
||||
close(db)
|
||||
close(outfile)
|
||||
|
||||
@@ -208,20 +208,6 @@ proc makeTest(test, options: string, cat: Category, action = actionCompile,
|
||||
|
||||
include categories
|
||||
|
||||
proc toJson(res: TResults): PJsonNode =
|
||||
result = newJObject()
|
||||
result["total"] = newJInt(res.total)
|
||||
result["passed"] = newJInt(res.passed)
|
||||
result["skipped"] = newJInt(res.skipped)
|
||||
|
||||
proc outputJson(reject, compile, run: TResults) =
|
||||
var doc = newJObject()
|
||||
doc["reject"] = toJson(reject)
|
||||
doc["compile"] = toJson(compile)
|
||||
doc["run"] = toJson(run)
|
||||
var s = pretty(doc)
|
||||
writeFile(jsonFile, s)
|
||||
|
||||
# proc runCaasTests(r: var TResults) =
|
||||
# for test, output, status, mode in caasTestsRunner():
|
||||
# r.addResult(test, "", output & "-> " & $mode,
|
||||
@@ -259,6 +245,7 @@ proc main() =
|
||||
var commit = 0
|
||||
discard parseInt(p.cmdLineRest.string, commit)
|
||||
generateHtml(resultsFile, commit)
|
||||
generateJson(jsonFile, commit)
|
||||
else:
|
||||
quit usage
|
||||
|
||||
|
||||
457
tests/tester.nim
457
tests/tester.nim
@@ -1,457 +0,0 @@
|
||||
#
|
||||
#
|
||||
# Nimrod Tester
|
||||
# (c) Copyright 2013 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
## This program verifies Nimrod against the testcases.
|
||||
|
||||
import
|
||||
parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json,
|
||||
marshal, cgi, parseopt #, caas
|
||||
|
||||
const
|
||||
cmdTemplate = r"nimrod cc --hints:on $# $#"
|
||||
resultsFile = "testresults.html"
|
||||
jsonFile = "testresults.json"
|
||||
Usage = "usage: tester [--print] " &
|
||||
"reject|compile|run|" &
|
||||
"merge|special|rodfiles| [nimrod options]\n" &
|
||||
" or: tester test|comp|rej singleTest"
|
||||
|
||||
type
|
||||
TTestAction = enum
|
||||
actionCompile, actionRun, actionReject
|
||||
TResultEnum = enum
|
||||
reNimrodcCrash, # nimrod compiler seems to have crashed
|
||||
reMsgsDiffer, # error messages differ
|
||||
reFilesDiffer, # expected and given filenames differ
|
||||
reLinesDiffer, # expected and given line numbers differ
|
||||
reOutputsDiffer,
|
||||
reExitcodesDiffer,
|
||||
reInvalidPeg,
|
||||
reCodegenFailure,
|
||||
reCodeNotFound,
|
||||
reExeNotFound,
|
||||
reIgnored, # test is ignored
|
||||
reSuccess # test was successful
|
||||
|
||||
TTarget = enum
|
||||
targetC, targetCpp, targetObjC, targetJS
|
||||
|
||||
TSpec = object
|
||||
action: TTestAction
|
||||
file, cmd: string
|
||||
outp: string
|
||||
line, exitCode: int
|
||||
msg: string
|
||||
ccodeCheck: string
|
||||
err: TResultEnum
|
||||
substr: bool
|
||||
TResults = object
|
||||
total, passed, skipped: int
|
||||
data: string
|
||||
|
||||
# ----------------------- Spec parser ----------------------------------------
|
||||
|
||||
when not defined(parseCfgBool):
|
||||
# candidate for the stdlib:
|
||||
proc parseCfgBool(s: string): bool =
|
||||
case normalize(s)
|
||||
of "y", "yes", "true", "1", "on": result = true
|
||||
of "n", "no", "false", "0", "off": result = false
|
||||
else: raise newException(EInvalidValue, "cannot interpret as a bool: " & s)
|
||||
|
||||
proc extractSpec(filename: string): string =
|
||||
const tripleQuote = "\"\"\""
|
||||
var x = readFile(filename).string
|
||||
var a = x.find(tripleQuote)
|
||||
var b = x.find(tripleQuote, a+3)
|
||||
# look for """ only in the first section
|
||||
if a >= 0 and b > a and a < 40:
|
||||
result = x.substr(a+3, b-1).replace("'''", tripleQuote)
|
||||
else:
|
||||
#echo "warning: file does not contain spec: " & filename
|
||||
result = ""
|
||||
|
||||
when not defined(nimhygiene):
|
||||
{.pragma: inject.}
|
||||
|
||||
template parseSpecAux(fillResult: stmt) {.immediate.} =
|
||||
var ss = newStringStream(extractSpec(filename))
|
||||
var p {.inject.}: TCfgParser
|
||||
open(p, ss, filename, 1)
|
||||
while true:
|
||||
var e {.inject.} = next(p)
|
||||
case e.kind
|
||||
of cfgEof: break
|
||||
of cfgSectionStart, cfgOption, cfgError:
|
||||
echo ignoreMsg(p, e)
|
||||
of cfgKeyValuePair:
|
||||
fillResult
|
||||
close(p)
|
||||
|
||||
proc parseSpec(filename: string): TSpec =
|
||||
result.file = filename
|
||||
result.msg = ""
|
||||
result.outp = ""
|
||||
result.ccodeCheck = ""
|
||||
result.cmd = cmdTemplate
|
||||
parseSpecAux:
|
||||
case normalize(e.key)
|
||||
of "action":
|
||||
case e.value.normalize
|
||||
of "compile": result.action = actionCompile
|
||||
of "run": result.action = actionRun
|
||||
of "reject": result.action = actionReject
|
||||
else: echo ignoreMsg(p, e)
|
||||
of "file": result.file = e.value
|
||||
of "line": discard parseInt(e.value, result.line)
|
||||
of "output": result.outp = e.value
|
||||
of "outputsub":
|
||||
result.outp = e.value
|
||||
result.substr = true
|
||||
of "exitcode":
|
||||
discard parseInt(e.value, result.exitCode)
|
||||
of "errormsg", "msg": result.msg = e.value
|
||||
of "disabled":
|
||||
if parseCfgBool(e.value): result.err = reIgnored
|
||||
of "cmd": result.cmd = e.value
|
||||
of "ccodecheck": result.ccodeCheck = e.value
|
||||
else: echo ignoreMsg(p, e)
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
let
|
||||
pegLineError =
|
||||
peg"{[^(]*} '(' {\d+} ', ' \d+ ') ' ('Error'/'Warning') ':' \s* {.*}"
|
||||
pegOtherError = peg"'Error:' \s* {.*}"
|
||||
pegSuccess = peg"'Hint: operation successful'.*"
|
||||
pegOfInterest = pegLineError / pegOtherError
|
||||
|
||||
proc callCompiler(cmdTemplate, filename, options: string): TSpec =
|
||||
let c = parseCmdLine(cmdTemplate % [options, filename])
|
||||
var p = startProcess(command=c[0], args=c[1.. -1],
|
||||
options={poStdErrToStdOut, poUseShell})
|
||||
let outp = p.outputStream
|
||||
var suc = ""
|
||||
var err = ""
|
||||
var x = newStringOfCap(120)
|
||||
while outp.readLine(x.TaintedString) or running(p):
|
||||
if x =~ pegOfInterest:
|
||||
# `err` should contain the last error/warning message
|
||||
err = x
|
||||
elif x =~ pegSuccess:
|
||||
suc = x
|
||||
close(p)
|
||||
result.msg = ""
|
||||
result.file = ""
|
||||
result.outp = ""
|
||||
result.line = -1
|
||||
if err =~ pegLineError:
|
||||
result.file = extractFilename(matches[0])
|
||||
result.line = parseInt(matches[1])
|
||||
result.msg = matches[2]
|
||||
elif err =~ pegOtherError:
|
||||
result.msg = matches[0]
|
||||
elif suc =~ pegSuccess:
|
||||
result.err = reSuccess
|
||||
|
||||
proc initResults: TResults =
|
||||
result.total = 0
|
||||
result.passed = 0
|
||||
result.skipped = 0
|
||||
result.data = ""
|
||||
|
||||
proc readResults(filename: string): TResults =
|
||||
result = marshal.to[TResults](readFile(filename).string)
|
||||
|
||||
proc writeResults(filename: string, r: TResults) =
|
||||
writeFile(filename, $$r)
|
||||
|
||||
proc `$`(x: TResults): string =
|
||||
result = ("Tests passed: $1 / $3 <br />\n" &
|
||||
"Tests skipped: $2 / $3 <br />\n") %
|
||||
[$x.passed, $x.skipped, $x.total]
|
||||
|
||||
proc colorResult(r: TResultEnum): string =
|
||||
case r
|
||||
of reIgnored: result = "<span style=\"color:fuchsia\">ignored</span>"
|
||||
of reSuccess: result = "<span style=\"color:green\">yes</span>"
|
||||
else: result = "<span style=\"color:red\">no</span>"
|
||||
|
||||
const
|
||||
TableHeader4 = "<table border=\"1\"><tr><td>Test</td><td>Expected</td>" &
|
||||
"<td>Given</td><td>Success</td></tr>\n"
|
||||
TableHeader3 = "<table border=\"1\"><tr><td>Test</td>" &
|
||||
"<td>Given</td><td>Success</td></tr>\n"
|
||||
TableFooter = "</table>\n"
|
||||
HtmlBegin = """<html>
|
||||
<head>
|
||||
<title>Test results</title>
|
||||
<style type="text/css">
|
||||
<!--""" & slurp("css/boilerplate.css") & "\n" &
|
||||
slurp("css/style.css") &
|
||||
"""-->
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>"""
|
||||
|
||||
HtmlEnd = "</body></html>"
|
||||
|
||||
proc td(s: string): string =
|
||||
result = s.substr(0, 200).XMLEncode
|
||||
|
||||
proc addResult(r: var TResults, test, expected, given: string,
|
||||
success: TResultEnum) =
|
||||
r.data.addf("<tr><td>$#</td><td>$#</td><td>$#</td><td>$#</td></tr>\n", [
|
||||
XMLEncode(test), td(expected), td(given), success.colorResult])
|
||||
|
||||
proc addResult(r: var TResults, test, given: string,
|
||||
success: TResultEnum) =
|
||||
r.data.addf("<tr><td>$#</td><td>$#</td><td>$#</td></tr>\n", [
|
||||
XMLEncode(test), td(given), success.colorResult])
|
||||
|
||||
proc listResults(reject, compile, run: TResults) =
|
||||
var s = HtmlBegin
|
||||
s.add("<h1>Tests to Reject</h1>\n")
|
||||
s.add($reject)
|
||||
s.add(TableHeader4 & reject.data & TableFooter)
|
||||
s.add("<br /><br /><br /><h1>Tests to Compile</h1>\n")
|
||||
s.add($compile)
|
||||
s.add(TableHeader3 & compile.data & TableFooter)
|
||||
s.add("<br /><br /><br /><h1>Tests to Run</h1>\n")
|
||||
s.add($run)
|
||||
s.add(TableHeader4 & run.data & TableFooter)
|
||||
s.add(HtmlEnd)
|
||||
writeFile(resultsFile, s)
|
||||
|
||||
proc cmpMsgs(r: var TResults, expected, given: TSpec, test: string) =
|
||||
if strip(expected.msg) notin strip(given.msg):
|
||||
r.addResult(test, expected.msg, given.msg, reMsgsDiffer)
|
||||
elif extractFilename(expected.file) != extractFilename(given.file) and
|
||||
"internal error:" notin expected.msg:
|
||||
r.addResult(test, expected.file, given.file, reFilesDiffer)
|
||||
elif expected.line != given.line and expected.line != 0:
|
||||
r.addResult(test, $expected.line, $given.line, reLinesDiffer)
|
||||
else:
|
||||
r.addResult(test, expected.msg, given.msg, reSuccess)
|
||||
inc(r.passed)
|
||||
|
||||
proc rejectSingleTest(r: var TResults, test, options: string) =
|
||||
let test = test.addFileExt(".nim")
|
||||
var t = extractFilename(test)
|
||||
inc(r.total)
|
||||
echo t
|
||||
var expected = parseSpec(test)
|
||||
if expected.err == reIgnored:
|
||||
r.addResult(t, "", "", reIgnored)
|
||||
inc(r.skipped)
|
||||
else:
|
||||
var given = callCompiler(expected.cmd, test, options)
|
||||
cmpMsgs(r, expected, given, t)
|
||||
|
||||
proc reject(r: var TResults, dir, options: string) =
|
||||
## handle all the tests that the compiler should reject
|
||||
for test in os.walkFiles(dir / "t*.nim"): rejectSingleTest(r, test, options)
|
||||
|
||||
proc codegenCheck(test, check, ext: string, given: var TSpec) =
|
||||
if check.len > 0:
|
||||
try:
|
||||
let (path, name, ext2) = test.splitFile
|
||||
echo path / "nimcache" / name.changeFileExt(ext)
|
||||
let contents = readFile(path / "nimcache" / name.changeFileExt(ext)).string
|
||||
if contents.find(check.peg) < 0:
|
||||
given.err = reCodegenFailure
|
||||
except EInvalidValue:
|
||||
given.err = reInvalidPeg
|
||||
except EIO:
|
||||
given.err = reCodeNotFound
|
||||
|
||||
proc codegenChecks(test: string, expected: TSpec, given: var TSpec) =
|
||||
codegenCheck(test, expected.ccodeCheck, ".c", given)
|
||||
|
||||
proc compile(r: var TResults, pattern, options: string) =
|
||||
for test in os.walkFiles(pattern):
|
||||
let t = extractFilename(test)
|
||||
echo t
|
||||
inc(r.total)
|
||||
let expected = parseSpec(test)
|
||||
if expected.err == reIgnored:
|
||||
r.addResult(t, "", reIgnored)
|
||||
inc(r.skipped)
|
||||
else:
|
||||
var given = callCompiler(expected.cmd, test, options)
|
||||
if given.err == reSuccess:
|
||||
codegenChecks(test, expected, given)
|
||||
r.addResult(t, given.msg, given.err)
|
||||
if given.err == reSuccess: inc(r.passed)
|
||||
|
||||
proc compileSingleTest(r: var TResults, test, options: string) =
|
||||
# does not extract the spec because the file is not supposed to have any
|
||||
let test = test.addFileExt(".nim")
|
||||
let t = extractFilename(test)
|
||||
inc(r.total)
|
||||
echo t
|
||||
let given = callCompiler(cmdTemplate, test, options)
|
||||
r.addResult(t, given.msg, given.err)
|
||||
if given.err == reSuccess: inc(r.passed)
|
||||
|
||||
proc runSingleTest(r: var TResults, test, options: string, target: TTarget) =
|
||||
var test = test.addFileExt(".nim")
|
||||
var t = extractFilename(test)
|
||||
echo t
|
||||
inc(r.total)
|
||||
var expected = parseSpec(test)
|
||||
if expected.err == reIgnored:
|
||||
r.addResult(t, "", "", reIgnored)
|
||||
inc(r.skipped)
|
||||
else:
|
||||
var given = callCompiler(expected.cmd, test, options)
|
||||
if given.err != reSuccess:
|
||||
r.addResult(t, "", given.msg, given.err)
|
||||
else:
|
||||
var exeFile: string
|
||||
if target == targetC:
|
||||
exeFile = changeFileExt(test, ExeExt)
|
||||
else:
|
||||
let (dir, file, ext) = splitFile(test)
|
||||
exeFile = dir / "nimcache" / file & ".js"
|
||||
|
||||
if existsFile(exeFile):
|
||||
var (buf, exitCode) = execCmdEx(
|
||||
(if target==targetJS: "node " else: "") & exeFile)
|
||||
if exitCode != expected.ExitCode:
|
||||
r.addResult(t, "exitcode: " & $expected.ExitCode,
|
||||
"exitcode: " & $exitCode, reExitCodesDiffer)
|
||||
else:
|
||||
if strip(buf.string) != strip(expected.outp):
|
||||
if not (expected.substr and expected.outp in buf.string):
|
||||
given.err = reOutputsDiffer
|
||||
if given.err == reSuccess:
|
||||
codeGenChecks(test, expected, given)
|
||||
if given.err == reSuccess: inc(r.passed)
|
||||
r.addResult(t, expected.outp, buf.string, given.err)
|
||||
else:
|
||||
r.addResult(t, expected.outp, "executable not found", reExeNotFound)
|
||||
|
||||
proc runSingleTest(r: var TResults, test, options: string) =
|
||||
runSingleTest(r, test, options, targetC)
|
||||
|
||||
proc run(r: var TResults, dir, options: string) =
|
||||
for test in os.walkFiles(dir / "t*.nim"): runSingleTest(r, test, options)
|
||||
|
||||
include specials
|
||||
|
||||
proc compileExample(r: var TResults, pattern, options: string) =
|
||||
for test in os.walkFiles(pattern): compileSingleTest(r, test, options)
|
||||
|
||||
proc toJson(res: TResults): PJsonNode =
|
||||
result = newJObject()
|
||||
result["total"] = newJInt(res.total)
|
||||
result["passed"] = newJInt(res.passed)
|
||||
result["skipped"] = newJInt(res.skipped)
|
||||
|
||||
proc outputJSON(reject, compile, run: TResults) =
|
||||
var doc = newJObject()
|
||||
doc["reject"] = toJson(reject)
|
||||
doc["compile"] = toJson(compile)
|
||||
doc["run"] = toJson(run)
|
||||
var s = pretty(doc)
|
||||
writeFile(jsonFile, s)
|
||||
|
||||
# proc runCaasTests(r: var TResults) =
|
||||
# for test, output, status, mode in caasTestsRunner():
|
||||
# r.addResult(test, "", output & "-> " & $mode,
|
||||
# if status: reSuccess else: reOutputsDiffer)
|
||||
|
||||
proc main() =
|
||||
os.putenv "NIMTEST_NO_COLOR", "1"
|
||||
os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
|
||||
|
||||
const
|
||||
compileJson = "compile.json"
|
||||
runJson = "run.json"
|
||||
rejectJson = "reject.json"
|
||||
|
||||
var optPrintResults = false
|
||||
var p = initOptParser()
|
||||
p.next()
|
||||
if p.kind == cmdLongoption:
|
||||
case p.key.string
|
||||
of "print": optPrintResults = true
|
||||
else: quit usage
|
||||
p.next()
|
||||
if p.kind != cmdArgument: quit usage
|
||||
var action = p.key.string.normalize
|
||||
p.next()
|
||||
var r = initResults()
|
||||
case action
|
||||
of "reject":
|
||||
reject(r, "tests/reject", p.cmdLineRest.string)
|
||||
rejectSpecialTests(r, p.cmdLineRest.string)
|
||||
writeResults(rejectJson, r)
|
||||
of "compile":
|
||||
compile(r, "tests/compile/t*.nim", p.cmdLineRest.string)
|
||||
compile(r, "tests/ccg/t*.nim", p.cmdLineRest.string)
|
||||
compile(r, "tests/js.nim", p.cmdLineRest.string)
|
||||
compileExample(r, "lib/pure/*.nim", p.cmdLineRest.string)
|
||||
compileExample(r, "examples/*.nim", p.cmdLineRest.string)
|
||||
compileExample(r, "examples/gtk/*.nim", p.cmdLineRest.string)
|
||||
compileExample(r, "examples/talk/*.nim", p.cmdLineRest.string)
|
||||
compileSpecialTests(r, p.cmdLineRest.string)
|
||||
writeResults(compileJson, r)
|
||||
of "run":
|
||||
run(r, "tests/run", p.cmdLineRest.string)
|
||||
runSpecialTests(r, p.cmdLineRest.string)
|
||||
writeResults(runJson, r)
|
||||
of "special":
|
||||
runSpecialTests(r, p.cmdLineRest.string)
|
||||
# runCaasTests(r)
|
||||
writeResults(runJson, r)
|
||||
of "rodfiles":
|
||||
runRodFiles(r, p.cmdLineRest.string)
|
||||
writeResults(runJson, r)
|
||||
of "js":
|
||||
if existsFile(runJSon):
|
||||
r = readResults(runJson)
|
||||
runJsTests(r, p.cmdLineRest.string)
|
||||
writeResults(runJson, r)
|
||||
of "merge":
|
||||
var rejectRes = readResults(rejectJson)
|
||||
var compileRes = readResults(compileJson)
|
||||
var runRes = readResults(runJson)
|
||||
listResults(rejectRes, compileRes, runRes)
|
||||
outputJSON(rejectRes, compileRes, runRes)
|
||||
of "dll":
|
||||
runDLLTests r, p.cmdLineRest.string
|
||||
of "gc":
|
||||
runGCTests(r, p.cmdLineRest.string)
|
||||
of "test":
|
||||
if p.kind != cmdArgument: quit usage
|
||||
var testFile = p.key.string
|
||||
p.next()
|
||||
runSingleTest(r, testFile, p.cmdLineRest.string)
|
||||
of "comp", "rej":
|
||||
if p.kind != cmdArgument: quit usage
|
||||
var testFile = p.key.string
|
||||
p.next()
|
||||
if peg"'/reject/'" in testFile or action == "rej":
|
||||
rejectSingleTest(r, testFile, p.cmdLineRest.string)
|
||||
elif peg"'/compile/'" in testFile or action == "comp":
|
||||
compileSingleTest(r, testFile, p.cmdLineRest.string)
|
||||
else:
|
||||
runSingleTest(r, testFile, p.cmdLineRest.string)
|
||||
else:
|
||||
quit usage
|
||||
|
||||
if optPrintResults: echo r, r.data
|
||||
|
||||
if paramCount() == 0:
|
||||
quit usage
|
||||
main()
|
||||
|
||||
5
todo.txt
5
todo.txt
@@ -1,14 +1,11 @@
|
||||
version 0.9.4
|
||||
=============
|
||||
|
||||
- better debugging support for writes to locations
|
||||
- document new templating symbol binding rules
|
||||
- fix eval in macros.nim
|
||||
|
||||
|
||||
Bugs
|
||||
====
|
||||
|
||||
- fix eval in macros.nim
|
||||
- new VM:
|
||||
- implement overflow checking
|
||||
- bug: 'type T = ref T' not recognized as illegal recursion
|
||||
|
||||
Reference in New Issue
Block a user