mirror of
https://github.com/nim-lang/Nim.git
synced 2026-08-29 18:11:44 +00:00
Testament pre parallel (#9137)
* testament: move to root dir (it's not a test)
* osproc: fix process index passed to afterRunEvent for parallel runs
it was passing the index of the process, not index of all commands
* testament: complete file move
(cherry picked from commit 97738a4f28)
This commit is contained in:
Jacek Sieka
75
testament/backend.nim
Normal file
75
testament/backend.nim
Normal file
@@ -0,0 +1,75 @@
|
||||
#
|
||||
#
|
||||
# The Nim Tester
|
||||
# (c) Copyright 2017 Andreas Rumpf
|
||||
#
|
||||
# Look at license.txt for more info.
|
||||
# All rights reserved.
|
||||
|
||||
import strutils, os, osproc, json
|
||||
|
||||
type
|
||||
MachineId* = distinct string
|
||||
CommitId = distinct string
|
||||
|
||||
proc `$`*(id: MachineId): string {.borrow.}
|
||||
#proc `$`(id: CommitId): string {.borrow.} # not used
|
||||
|
||||
var
|
||||
thisMachine: MachineId
|
||||
thisCommit: CommitId
|
||||
thisBranch: string
|
||||
|
||||
{.experimental.}
|
||||
proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip
|
||||
|
||||
proc getMachine*(): MachineId =
|
||||
var name = "hostname"()
|
||||
if name.len == 0:
|
||||
name = when defined(posix): getenv"HOSTNAME".string
|
||||
else: getenv"COMPUTERNAME".string
|
||||
if name.len == 0:
|
||||
quit "cannot determine the machine name"
|
||||
|
||||
result = MachineId(name)
|
||||
|
||||
proc getCommit(): CommitId =
|
||||
const commLen = "commit ".len
|
||||
let hash = "git log -n 1"()[commLen..commLen+10]
|
||||
thisBranch = "git symbolic-ref --short HEAD"()
|
||||
if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD"
|
||||
result = CommitId(hash)
|
||||
|
||||
var
|
||||
results: File
|
||||
currentCategory: string
|
||||
entries: int
|
||||
|
||||
proc writeTestResult*(name, category, target,
|
||||
action, result, expected, given: string) =
|
||||
createDir("testresults")
|
||||
if currentCategory != category:
|
||||
if currentCategory.len > 0:
|
||||
results.writeLine("]")
|
||||
close(results)
|
||||
currentCategory = category
|
||||
results = open("testresults" / category.addFileExt"json", fmWrite)
|
||||
results.writeLine("[")
|
||||
entries = 0
|
||||
|
||||
let jentry = %*{"name": name, "category": category, "target": target,
|
||||
"action": action, "result": result, "expected": expected, "given": given,
|
||||
"machine": thisMachine.string, "commit": thisCommit.string, "branch": thisBranch}
|
||||
if entries > 0:
|
||||
results.writeLine(",")
|
||||
results.write($jentry)
|
||||
inc entries
|
||||
|
||||
proc open*() =
|
||||
thisMachine = getMachine()
|
||||
thisCommit = getCommit()
|
||||
|
||||
proc close*() =
|
||||
if currentCategory.len > 0:
|
||||
results.writeLine("]")
|
||||
close(results)
|
||||
195
testament/caasdriver.nim
Normal file
195
testament/caasdriver.nim
Normal file
@@ -0,0 +1,195 @@
|
||||
import osproc, streams, os, strutils, re
|
||||
{.experimental.}
|
||||
|
||||
## Compiler as a service tester.
|
||||
##
|
||||
## Please read docs/idetools.txt for information about this.
|
||||
|
||||
|
||||
type
|
||||
TRunMode = enum
|
||||
ProcRun, CaasRun, SymbolProcRun
|
||||
|
||||
NimSession* = object
|
||||
nim: Process # Holds the open process for CaasRun sessions, nil otherwise.
|
||||
mode: TRunMode # Stores the type of run mode the session was started with.
|
||||
lastOutput: string # Preserves the last output, needed for ProcRun mode.
|
||||
filename: string # Appended to each command starting with '>'. Also a var.
|
||||
modname: string # Like filename but without extension.
|
||||
nimcache: string # Input script based name for the nimcache dir.
|
||||
|
||||
const
|
||||
modes = [CaasRun, ProcRun, SymbolProcRun]
|
||||
filenameReplaceVar = "$TESTNIM"
|
||||
moduleReplaceVar = "$MODULE"
|
||||
silentReplaceVar = "$SILENT"
|
||||
silentReplaceText = "--verbosity:0 --hints:off"
|
||||
|
||||
var
|
||||
TesterDir = getAppDir() / ".."
|
||||
NimBin = TesterDir / "../bin/nim"
|
||||
|
||||
proc replaceVars(session: var NimSession, text: string): string =
|
||||
result = text.replace(filenameReplaceVar, session.filename)
|
||||
result = result.replace(moduleReplaceVar, session.modname)
|
||||
result = result.replace(silentReplaceVar, silentReplaceText)
|
||||
|
||||
proc startNimSession(project, script: string, mode: TRunMode):
|
||||
NimSession =
|
||||
let (dir, name, ext) = project.splitFile
|
||||
result.mode = mode
|
||||
result.lastOutput = ""
|
||||
result.filename = name & ext
|
||||
result.modname = name
|
||||
|
||||
let (nimcacheDir, nimcacheName, nimcacheExt) = script.splitFile
|
||||
result.nimcache = "SymbolProcRun." & nimcacheName
|
||||
|
||||
if mode == SymbolProcRun:
|
||||
removeDir(nimcacheDir / result.nimcache)
|
||||
else:
|
||||
removeDir(nimcacheDir / "nimcache")
|
||||
|
||||
if mode == CaasRun:
|
||||
result.nim = startProcess(NimBin, workingDir = dir,
|
||||
args = ["serve", "--server.type:stdin", name])
|
||||
|
||||
proc doCaasCommand(session: var NimSession, command: string): string =
|
||||
assert session.mode == CaasRun
|
||||
session.nim.inputStream.write(session.replaceVars(command) & "\n")
|
||||
session.nim.inputStream.flush
|
||||
|
||||
result = ""
|
||||
|
||||
while true:
|
||||
var line = TaintedString("")
|
||||
if session.nim.outputStream.readLine(line):
|
||||
if line.string == "": break
|
||||
result.add(line.string & "\n")
|
||||
else:
|
||||
result = "FAILED TO EXECUTE: " & command & "\n" & result
|
||||
break
|
||||
|
||||
proc doProcCommand(session: var NimSession, command: string): string =
|
||||
try:
|
||||
assert session.mode == ProcRun or session.mode == SymbolProcRun
|
||||
except:
|
||||
result = "FAILED TO EXECUTE: " & command & "\n" & result
|
||||
var
|
||||
process = startProcess(NimBin, args = session.replaceVars(command).split)
|
||||
stream = outputStream(process)
|
||||
line = TaintedString("")
|
||||
|
||||
result = ""
|
||||
while stream.readLine(line):
|
||||
if result.len > 0: result &= "\n"
|
||||
result &= line.string
|
||||
|
||||
process.close()
|
||||
|
||||
proc doCommand(session: var NimSession, command: string) =
|
||||
if session.mode == CaasRun:
|
||||
if not session.nim.running:
|
||||
session.lastOutput = "FAILED TO EXECUTE: " & command & "\n" &
|
||||
"Exit code " & $session.nim.peekExitCode
|
||||
return
|
||||
session.lastOutput = doCaasCommand(session,
|
||||
command & " " & session.filename)
|
||||
else:
|
||||
var command = command
|
||||
# For symbol runs we prepend the necessary parameters to avoid clobbering
|
||||
# the normal nimcache.
|
||||
if session.mode == SymbolProcRun:
|
||||
command = "--symbolFiles:on --nimcache:" & session.nimcache &
|
||||
" " & command
|
||||
session.lastOutput = doProcCommand(session,
|
||||
command & " " & session.filename)
|
||||
|
||||
proc destroy(session: var NimSession) {.destructor.} =
|
||||
if session.mode == CaasRun:
|
||||
session.nim.close
|
||||
|
||||
proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): bool =
|
||||
result = true
|
||||
|
||||
var f = open(script)
|
||||
var project = TaintedString("")
|
||||
|
||||
if f.readLine(project):
|
||||
var
|
||||
s = startNimSession(script.parentDir / project.string, script, mode)
|
||||
tline = TaintedString("")
|
||||
ln = 1
|
||||
|
||||
while f.readLine(tline):
|
||||
var line = tline.string
|
||||
inc ln
|
||||
|
||||
# Filter lines by run mode, removing the prefix if the mode is current.
|
||||
for testMode in modes:
|
||||
if line.startsWith($testMode):
|
||||
if testMode != mode:
|
||||
line = ""
|
||||
else:
|
||||
line = line[len($testMode)..len(line) - 1].strip
|
||||
break
|
||||
|
||||
if line.strip.len == 0: continue
|
||||
|
||||
if line.startsWith("#"):
|
||||
output.writeLine line
|
||||
continue
|
||||
elif line.startsWith(">"):
|
||||
s.doCommand(line.substr(1).strip)
|
||||
output.writeLine line, "\n", if verbose: s.lastOutput else: ""
|
||||
else:
|
||||
var expectMatch = true
|
||||
var pattern = s.replaceVars(line)
|
||||
if line.startsWith("!"):
|
||||
pattern = pattern.substr(1).strip
|
||||
expectMatch = false
|
||||
|
||||
let actualMatch =
|
||||
s.lastOutput.find(re(pattern, flags = {reStudy})) != -1
|
||||
|
||||
if expectMatch == actualMatch:
|
||||
output.writeLine "SUCCESS ", line
|
||||
else:
|
||||
output.writeLine "FAILURE ", line
|
||||
result = false
|
||||
|
||||
iterator caasTestsRunner*(filter = "", verbose = false): tuple[test,
|
||||
output: string, status: bool,
|
||||
mode: TRunMode] =
|
||||
for scenario in os.walkFiles(TesterDir / "caas/*.txt"):
|
||||
if filter.len > 0 and find(scenario, filter) == -1: continue
|
||||
for mode in modes:
|
||||
var outStream = newStringStream()
|
||||
let r = doScenario(scenario, outStream, mode, verbose)
|
||||
yield (scenario, outStream.data, r, mode)
|
||||
|
||||
when isMainModule:
|
||||
var
|
||||
filter = ""
|
||||
failures = 0
|
||||
verbose = false
|
||||
|
||||
for i in 0..paramCount() - 1:
|
||||
let param = string(paramStr(i + 1))
|
||||
case param
|
||||
of "verbose": verbose = true
|
||||
else: filter = param
|
||||
|
||||
if verbose and len(filter) > 0:
|
||||
echo "Running only test cases matching filter '$1'" % [filter]
|
||||
|
||||
for test, output, result, mode in caasTestsRunner(filter, verbose):
|
||||
if not result or verbose:
|
||||
echo "Mode ", $mode, " (", if result: "succeeded)" else: "failed)"
|
||||
echo test
|
||||
echo output
|
||||
echo "---------\n"
|
||||
if not result:
|
||||
failures += 1
|
||||
|
||||
quit(failures)
|
||||
539
testament/categories.nim
Normal file
539
testament/categories.nim
Normal file
@@ -0,0 +1,539 @@
|
||||
#
|
||||
#
|
||||
# Nim Tester
|
||||
# (c) Copyright 2015 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
## Include for the tester that contains test suites that test special features
|
||||
## of the compiler.
|
||||
|
||||
# included from tester.nim
|
||||
# ---------------- ROD file tests ---------------------------------------------
|
||||
|
||||
const
|
||||
rodfilesDir = "tests/rodfiles"
|
||||
|
||||
proc delNimCache(filename, options: string) =
|
||||
for target in low(TTarget)..high(TTarget):
|
||||
let dir = nimcacheDir(filename, options, target)
|
||||
try:
|
||||
removeDir(dir)
|
||||
except OSError:
|
||||
echo "[Warning] could not delete: ", dir
|
||||
|
||||
proc runRodFiles(r: var TResults, cat: Category, options: string) =
|
||||
template test(filename: string, clearCacheFirst=false) =
|
||||
if clearCacheFirst: delNimCache(filename, options)
|
||||
testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun)
|
||||
|
||||
|
||||
# test basic recompilation scheme:
|
||||
test "hallo", true
|
||||
test "hallo"
|
||||
when false:
|
||||
# test incremental type information:
|
||||
test "hallo2"
|
||||
|
||||
# test type converters:
|
||||
test "aconv", true
|
||||
test "bconv"
|
||||
|
||||
# test G, A, B example from the documentation; test init sections:
|
||||
test "deada", true
|
||||
test "deada2"
|
||||
|
||||
when false:
|
||||
# test method generation:
|
||||
test "bmethods", true
|
||||
test "bmethods2"
|
||||
|
||||
# test generics:
|
||||
test "tgeneric1", true
|
||||
test "tgeneric2"
|
||||
|
||||
proc compileRodFiles(r: var TResults, cat: Category, options: string) =
|
||||
template test(filename: untyped, clearCacheFirst=true) =
|
||||
if clearCacheFirst: delNimCache(filename, options)
|
||||
testSpec r, makeTest(rodfilesDir / filename, options, cat)
|
||||
|
||||
# test DLL interfacing:
|
||||
test "gtkex1", true
|
||||
test "gtkex2"
|
||||
|
||||
# --------------------- flags tests -------------------------------------------
|
||||
|
||||
proc flagTests(r: var TResults, cat: Category, options: string) =
|
||||
# --genscript
|
||||
const filename = "tests"/"flags"/"tgenscript"
|
||||
const genopts = " --genscript"
|
||||
let nimcache = nimcacheDir(filename, genopts, targetC)
|
||||
testSpec r, makeTest(filename, genopts, cat)
|
||||
|
||||
when defined(windows):
|
||||
testExec r, makeTest(filename, " cmd /c cd " & nimcache &
|
||||
" && compile_tgenscript.bat", cat)
|
||||
|
||||
elif defined(posix):
|
||||
testExec r, makeTest(filename, " sh -c \"cd " & nimcache &
|
||||
" && sh compile_tgenscript.sh\"", cat)
|
||||
|
||||
# Run
|
||||
testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat)
|
||||
|
||||
# --------------------- DLL generation tests ----------------------------------
|
||||
|
||||
proc safeCopyFile(src, dest: string) =
|
||||
try:
|
||||
copyFile(src, dest)
|
||||
except OSError:
|
||||
echo "[Warning] could not copy: ", src, " to ", dest
|
||||
|
||||
proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
|
||||
const rpath = when defined(macosx):
|
||||
" --passL:-rpath --passL:@loader_path"
|
||||
else:
|
||||
""
|
||||
|
||||
testSpec c, makeTest("lib/nimrtl.nim",
|
||||
options & " --app:lib -d:createNimRtl --threads:on", cat)
|
||||
testSpec c, makeTest("tests/dll/server.nim",
|
||||
options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat)
|
||||
|
||||
|
||||
when defined(Windows):
|
||||
# windows looks in the dir of the exe (yay!):
|
||||
var nimrtlDll = DynlibFormat % "nimrtl"
|
||||
safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
|
||||
else:
|
||||
# posix relies on crappy LD_LIBRARY_PATH (ugh!):
|
||||
const libpathenv = when defined(haiku):
|
||||
"LIBRARY_PATH"
|
||||
else:
|
||||
"LD_LIBRARY_PATH"
|
||||
var libpath = getEnv(libpathenv).string
|
||||
# Temporarily add the lib directory to LD_LIBRARY_PATH:
|
||||
putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: ""))
|
||||
defer: putEnv(libpathenv, libpath)
|
||||
var nimrtlDll = DynlibFormat % "nimrtl"
|
||||
safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
|
||||
|
||||
testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl --threads:on" & rpath,
|
||||
cat, actionRun)
|
||||
|
||||
proc dllTests(r: var TResults, cat: Category, options: string) =
|
||||
# dummy compile result:
|
||||
var c = initResults()
|
||||
|
||||
runBasicDLLTest c, r, cat, options
|
||||
runBasicDLLTest c, r, cat, options & " -d:release"
|
||||
when not defined(windows):
|
||||
# still cannot find a recent Windows version of boehm.dll:
|
||||
runBasicDLLTest c, r, cat, options & " --gc:boehm"
|
||||
runBasicDLLTest c, r, cat, options & " -d:release --gc:boehm"
|
||||
|
||||
# ------------------------------ GC tests -------------------------------------
|
||||
|
||||
proc gcTests(r: var TResults, cat: Category, options: string) =
|
||||
template testWithNone(filename: untyped) =
|
||||
testSpec r, makeTest("tests/gc" / filename, options &
|
||||
" --gc:none", cat, actionRun)
|
||||
testSpec r, makeTest("tests/gc" / filename, options &
|
||||
" -d:release --gc:none", cat, actionRun)
|
||||
|
||||
template testWithoutMs(filename: untyped) =
|
||||
testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun)
|
||||
testSpec r, makeTest("tests/gc" / filename, options &
|
||||
" -d:release", cat, actionRun)
|
||||
testSpec r, makeTest("tests/gc" / filename, options &
|
||||
" -d:release -d:useRealtimeGC", cat, actionRun)
|
||||
|
||||
template testWithoutBoehm(filename: untyped) =
|
||||
testWithoutMs filename
|
||||
testSpec r, makeTest("tests/gc" / filename, options &
|
||||
" --gc:markAndSweep", cat, actionRun)
|
||||
testSpec r, makeTest("tests/gc" / filename, options &
|
||||
" -d:release --gc:markAndSweep", cat, actionRun)
|
||||
template test(filename: untyped) =
|
||||
testWithoutBoehm filename
|
||||
when not defined(windows) and not defined(android):
|
||||
# AR: cannot find any boehm.dll on the net, right now, so disabled
|
||||
# for windows:
|
||||
testSpec r, makeTest("tests/gc" / filename, options &
|
||||
" --gc:boehm", cat, actionRun)
|
||||
testSpec r, makeTest("tests/gc" / filename, options &
|
||||
" -d:release --gc:boehm", cat, actionRun)
|
||||
|
||||
testWithoutBoehm "foreign_thr"
|
||||
test "gcemscripten"
|
||||
test "growobjcrash"
|
||||
test "gcbench"
|
||||
test "gcleak"
|
||||
test "gcleak2"
|
||||
testWithoutBoehm "gctest"
|
||||
testWithNone "gctest"
|
||||
test "gcleak3"
|
||||
test "gcleak4"
|
||||
# Disabled because it works and takes too long to run:
|
||||
#test "gcleak5"
|
||||
testWithoutBoehm "weakrefs"
|
||||
test "cycleleak"
|
||||
testWithoutBoehm "closureleak"
|
||||
testWithoutMs "refarrayleak"
|
||||
|
||||
testWithoutBoehm "tlists"
|
||||
testWithoutBoehm "thavlak"
|
||||
|
||||
test "stackrefleak"
|
||||
test "cyclecollector"
|
||||
|
||||
proc longGCTests(r: var TResults, cat: Category, options: string) =
|
||||
when defined(windows):
|
||||
let cOptions = "-ldl -DWIN"
|
||||
else:
|
||||
let cOptions = "-ldl"
|
||||
|
||||
var c = initResults()
|
||||
# According to ioTests, this should compile the file
|
||||
testNoSpec c, makeTest("tests/realtimeGC/shared", options, cat, actionCompile)
|
||||
testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat, actionRun)
|
||||
testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat, actionRun)
|
||||
|
||||
# ------------------------- threading tests -----------------------------------
|
||||
|
||||
proc threadTests(r: var TResults, cat: Category, options: string) =
|
||||
template test(filename: untyped) =
|
||||
testSpec r, makeTest(filename, options, cat, actionRun)
|
||||
testSpec r, makeTest(filename, options & " -d:release", cat, actionRun)
|
||||
testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat, actionRun)
|
||||
for t in os.walkFiles("tests/threads/t*.nim"):
|
||||
test(t)
|
||||
|
||||
# ------------------------- IO tests ------------------------------------------
|
||||
|
||||
proc ioTests(r: var TResults, cat: Category, options: string) =
|
||||
# We need readall_echo to be compiled for this test to run.
|
||||
# dummy compile result:
|
||||
var c = initResults()
|
||||
testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat)
|
||||
testSpec r, makeTest("tests/system/tio", options, cat)
|
||||
|
||||
# ------------------------- async tests ---------------------------------------
|
||||
proc asyncTests(r: var TResults, cat: Category, options: string) =
|
||||
template test(filename: untyped) =
|
||||
testSpec r, makeTest(filename, options, cat)
|
||||
for t in os.walkFiles("tests/async/t*.nim"):
|
||||
test(t)
|
||||
|
||||
# ------------------------- debugger tests ------------------------------------
|
||||
|
||||
proc debuggerTests(r: var TResults, cat: Category, options: string) =
|
||||
testNoSpec r, makeTest("tools/nimgrep", options & " --debugger:on", cat)
|
||||
|
||||
# ------------------------- JS tests ------------------------------------------
|
||||
|
||||
proc jsTests(r: var TResults, cat: Category, options: string) =
|
||||
template test(filename: untyped) =
|
||||
testSpec r, makeTest(filename, options & " -d:nodejs", cat,
|
||||
actionRun), targetJS
|
||||
testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat,
|
||||
actionRun), targetJS
|
||||
|
||||
for t in os.walkFiles("tests/js/t*.nim"):
|
||||
test(t)
|
||||
for testfile in ["exception/texceptions", "exception/texcpt1",
|
||||
"exception/texcsub", "exception/tfinally",
|
||||
"exception/tfinally2", "exception/tfinally3",
|
||||
"exception/tunhandledexc",
|
||||
"actiontable/tactiontable", "method/tmultim1",
|
||||
"method/tmultim3", "method/tmultim4",
|
||||
"varres/tvarres0", "varres/tvarres3", "varres/tvarres4",
|
||||
"varres/tvartup", "misc/tints", "misc/tunsignedinc",
|
||||
"async/tjsandnativeasync"]:
|
||||
test "tests/" & testfile & ".nim"
|
||||
|
||||
for testfile in ["strutils", "json", "random", "times", "logging"]:
|
||||
test "lib/pure/" & testfile & ".nim"
|
||||
|
||||
# ------------------------- nim in action -----------
|
||||
|
||||
proc testNimInAction(r: var TResults, cat: Category, options: string) =
|
||||
let options = options & " --nilseqs:on"
|
||||
|
||||
template test(filename: untyped, action: untyped) =
|
||||
testSpec r, makeTest(filename, options, cat, action)
|
||||
|
||||
template testJS(filename: untyped) =
|
||||
testSpec r, makeTest(filename, options, cat, actionCompile), targetJS
|
||||
|
||||
template testCPP(filename: untyped) =
|
||||
testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP
|
||||
|
||||
let tests = [
|
||||
"niminaction/Chapter1/various1",
|
||||
"niminaction/Chapter2/various2",
|
||||
"niminaction/Chapter2/resultaccept",
|
||||
"niminaction/Chapter2/resultreject",
|
||||
"niminaction/Chapter2/explicit_discard",
|
||||
"niminaction/Chapter2/no_def_eq",
|
||||
"niminaction/Chapter2/no_iterator",
|
||||
"niminaction/Chapter2/no_seq_type",
|
||||
"niminaction/Chapter3/ChatApp/src/server",
|
||||
"niminaction/Chapter3/ChatApp/src/client",
|
||||
"niminaction/Chapter3/various3",
|
||||
"niminaction/Chapter6/WikipediaStats/concurrency_regex",
|
||||
"niminaction/Chapter6/WikipediaStats/concurrency",
|
||||
"niminaction/Chapter6/WikipediaStats/naive",
|
||||
"niminaction/Chapter6/WikipediaStats/parallel_counts",
|
||||
"niminaction/Chapter6/WikipediaStats/race_condition",
|
||||
"niminaction/Chapter6/WikipediaStats/sequential_counts",
|
||||
"niminaction/Chapter6/WikipediaStats/unguarded_access",
|
||||
"niminaction/Chapter7/Tweeter/src/tweeter",
|
||||
"niminaction/Chapter7/Tweeter/src/createDatabase",
|
||||
"niminaction/Chapter7/Tweeter/tests/database_test",
|
||||
"niminaction/Chapter8/sdl/sdl_test"
|
||||
]
|
||||
|
||||
# Verify that the files have not been modified. Death shall fall upon
|
||||
# whoever edits these hashes without dom96's permission, j/k. But please only
|
||||
# edit when making a conscious breaking change, also please try to make your
|
||||
# commit message clear and notify me so I can easily compile an errata later.
|
||||
var testHashes: seq[string] = @[]
|
||||
|
||||
for test in tests:
|
||||
testHashes.add(getMD5(readFile("tests" / test.addFileExt("nim")).string))
|
||||
|
||||
const refHashes = @[
|
||||
"51afdfa84b3ca3d810809d6c4e5037ba", "30f07e4cd5eaec981f67868d4e91cfcf",
|
||||
"d14e7c032de36d219c9548066a97e846", "2e40bfd5daadb268268727da91bb4e81",
|
||||
"c5d3853ed0aba04bf6d35ba28a98dca0", "058603145ff92d46c009006b06e5b228",
|
||||
"7b94a029b94ddb7efafddd546c965ff6", "586d74514394e49f2370dfc01dd9e830",
|
||||
"e1901837b757c9357dc8d259fd0ef0f6", "097670c7ae12e825debaf8ec3995227b",
|
||||
"a8cb7b78cc78d28535ab467361db5d6e", "bfaec2816a1848991b530c1ad17a0184",
|
||||
"47cb71bb4c1198d6d29cdbee05aa10b9", "87e4436809f9d73324cfc4f57f116770",
|
||||
"7b7db5cddc8cf8fa9b6776eef1d0a31d", "e6e40219f0f2b877869b738737b7685e",
|
||||
"6532ee87d819f2605a443d5e94f9422a", "9a8fe78c588d08018843b64b57409a02",
|
||||
"03a801275b8b76b4170c870cd0da079d", "20bb7d3e2d38d43b0cb5fcff4909a4a8",
|
||||
"af6844598f534fab6942abfa4dfe9ab2", "2a7a17f84f6503d9bc89a5ab8feea127"
|
||||
]
|
||||
doAssert testHashes == refHashes, "Nim in Action tests were changed."
|
||||
|
||||
# Run the tests.
|
||||
for testfile in tests:
|
||||
test "tests/" & testfile & ".nim", actionCompile
|
||||
|
||||
let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim"
|
||||
testJS jsFile
|
||||
|
||||
let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim"
|
||||
testCPP cppFile
|
||||
|
||||
|
||||
|
||||
|
||||
# ------------------------- manyloc -------------------------------------------
|
||||
#proc runSpecialTests(r: var TResults, options: string) =
|
||||
# for t in ["lib/packages/docutils/highlite"]:
|
||||
# testSpec(r, t, options)
|
||||
|
||||
proc findMainFile(dir: string): string =
|
||||
# finds the file belonging to ".nim.cfg"; if there is no such file
|
||||
# it returns the some ".nim" file if there is only one:
|
||||
const cfgExt = ".nim.cfg"
|
||||
result = ""
|
||||
var nimFiles = 0
|
||||
for kind, file in os.walkDir(dir):
|
||||
if kind == pcFile:
|
||||
if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim"
|
||||
elif file.endsWith(".nim"):
|
||||
if result.len == 0: result = file
|
||||
inc nimFiles
|
||||
if nimFiles != 1: result.setlen(0)
|
||||
|
||||
proc manyLoc(r: var TResults, cat: Category, options: string) =
|
||||
for kind, dir in os.walkDir("tests/manyloc"):
|
||||
if kind == pcDir:
|
||||
when defined(windows):
|
||||
if dir.endsWith"nake": continue
|
||||
if dir.endsWith"named_argument_bug": continue
|
||||
let mainfile = findMainFile(dir)
|
||||
if mainfile != "":
|
||||
testNoSpec r, makeTest(mainfile, options, cat)
|
||||
|
||||
proc compileExample(r: var TResults, pattern, options: string, cat: Category) =
|
||||
for test in os.walkFiles(pattern):
|
||||
testNoSpec r, makeTest(test, options, cat)
|
||||
|
||||
proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
|
||||
for test in os.walkFiles(pattern):
|
||||
let name = extractFilename(test)
|
||||
if name notin disabledFiles:
|
||||
let contents = readFile(test).string
|
||||
if contents.contains("when isMainModule"):
|
||||
testSpec r, makeTest(test, options, cat, actionRunNoSpec)
|
||||
else:
|
||||
testNoSpec r, makeTest(test, options, cat, actionCompile)
|
||||
|
||||
# ----------------------------- nimble ----------------------------------------
|
||||
type PackageFilter = enum
|
||||
pfCoreOnly
|
||||
pfExtraOnly
|
||||
pfAll
|
||||
|
||||
var nimbleDir = getEnv("NIMBLE_DIR").string
|
||||
if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
|
||||
let
|
||||
nimbleExe = findExe("nimble")
|
||||
#packageDir = nimbleDir / "pkgs" # not used
|
||||
packageIndex = nimbleDir / "packages.json"
|
||||
|
||||
proc waitForExitEx(p: Process): int =
|
||||
var outp = outputStream(p)
|
||||
var line = newStringOfCap(120).TaintedString
|
||||
while true:
|
||||
if outp.readLine(line):
|
||||
discard
|
||||
else:
|
||||
result = peekExitCode(p)
|
||||
if result != -1: break
|
||||
close(p)
|
||||
|
||||
proc getPackageDir(package: string): string =
|
||||
## TODO - Replace this with dom's version comparison magic.
|
||||
var commandOutput = execCmdEx("nimble path $#" % package)
|
||||
if commandOutput.exitCode != QuitSuccess:
|
||||
return ""
|
||||
else:
|
||||
result = commandOutput[0].string
|
||||
|
||||
iterator listPackages(filter: PackageFilter): tuple[name, url: string] =
|
||||
let packageList = parseFile(packageIndex)
|
||||
|
||||
for package in packageList.items():
|
||||
let
|
||||
name = package["name"].str
|
||||
url = package["url"].str
|
||||
isCorePackage = "nim-lang" in normalize(url)
|
||||
case filter:
|
||||
of pfCoreOnly:
|
||||
if isCorePackage:
|
||||
yield (name, url)
|
||||
of pfExtraOnly:
|
||||
if not isCorePackage:
|
||||
yield (name, url)
|
||||
of pfAll:
|
||||
yield (name, url)
|
||||
|
||||
proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
|
||||
if nimbleExe == "":
|
||||
echo("[Warning] - Cannot run nimble tests: Nimble binary not found.")
|
||||
return
|
||||
|
||||
if execCmd("$# update" % nimbleExe) == QuitFailure:
|
||||
echo("[Warning] - Cannot run nimble tests: Nimble update failed.")
|
||||
return
|
||||
|
||||
let packageFileTest = makeTest("PackageFileParsed", "", cat)
|
||||
try:
|
||||
for name, url in listPackages(filter):
|
||||
var test = makeTest(name, "", cat)
|
||||
echo(url)
|
||||
let
|
||||
installProcess = startProcess(nimbleExe, "", ["install", "-y", name])
|
||||
installStatus = waitForExitEx(installProcess)
|
||||
installProcess.close
|
||||
if installStatus != QuitSuccess:
|
||||
r.addResult(test, targetC, "", "", reInstallFailed)
|
||||
continue
|
||||
|
||||
let
|
||||
buildPath = getPackageDir(name).strip
|
||||
buildProcess = startProcess(nimbleExe, buildPath, ["build"])
|
||||
buildStatus = waitForExitEx(buildProcess)
|
||||
buildProcess.close
|
||||
if buildStatus != QuitSuccess:
|
||||
r.addResult(test, targetC, "", "", reBuildFailed)
|
||||
r.addResult(test, targetC, "", "", reSuccess)
|
||||
r.addResult(packageFileTest, targetC, "", "", reSuccess)
|
||||
except JsonParsingError:
|
||||
echo("[Warning] - Cannot run nimble tests: Invalid package file.")
|
||||
r.addResult(packageFileTest, targetC, "", "", reBuildFailed)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
const AdditionalCategories = ["debugger", "examples", "lib"]
|
||||
|
||||
proc `&.?`(a, b: string): string =
|
||||
# candidate for the stdlib?
|
||||
result = if b.startswith(a): b else: a & b
|
||||
|
||||
#proc `&?.`(a, b: string): string = # not used
|
||||
# candidate for the stdlib?
|
||||
#result = if a.endswith(b): a else: a & b
|
||||
|
||||
proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
|
||||
let test = "tests" & DirSep &.? cat.string / test
|
||||
let target = if cat.string.normalize == "js": targetJS else: targetC
|
||||
|
||||
if existsFile(test): testSpec r, makeTest(test, options, cat), target
|
||||
else: echo "[Warning] - ", test, " test does not exist"
|
||||
|
||||
proc processCategory(r: var TResults, cat: Category, options: string) =
|
||||
case cat.string.normalize
|
||||
of "rodfiles":
|
||||
when false:
|
||||
compileRodFiles(r, cat, options)
|
||||
runRodFiles(r, cat, options)
|
||||
of "js":
|
||||
# only run the JS tests on Windows or Linux because Travis is bad
|
||||
# and other OSes like Haiku might lack nodejs:
|
||||
if not defined(linux) and isTravis:
|
||||
discard
|
||||
else:
|
||||
jsTests(r, cat, options)
|
||||
of "dll":
|
||||
dllTests(r, cat, options)
|
||||
of "flags":
|
||||
flagTests(r, cat, options)
|
||||
of "gc":
|
||||
gcTests(r, cat, options)
|
||||
of "longgc":
|
||||
longGCTests(r, cat, options)
|
||||
of "debugger":
|
||||
debuggerTests(r, cat, options)
|
||||
of "manyloc":
|
||||
manyLoc r, cat, options
|
||||
of "threads":
|
||||
threadTests r, cat, options & " --threads:on"
|
||||
of "io":
|
||||
ioTests r, cat, options
|
||||
of "async":
|
||||
asyncTests r, cat, options
|
||||
of "lib":
|
||||
testStdlib(r, "lib/pure/*.nim", options, cat)
|
||||
testStdlib(r, "lib/packages/docutils/highlite", options, cat)
|
||||
of "examples":
|
||||
compileExample(r, "examples/*.nim", options, cat)
|
||||
compileExample(r, "examples/gtk/*.nim", options, cat)
|
||||
compileExample(r, "examples/talk/*.nim", options, cat)
|
||||
of "nimble-core":
|
||||
testNimblePackages(r, cat, pfCoreOnly)
|
||||
of "nimble-extra":
|
||||
testNimblePackages(r, cat, pfExtraOnly)
|
||||
of "nimble-all":
|
||||
testNimblePackages(r, cat, pfAll)
|
||||
of "niminaction":
|
||||
testNimInAction(r, cat, options)
|
||||
of "untestable":
|
||||
# We can't test it because it depends on a third party.
|
||||
discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
|
||||
else:
|
||||
var testsRun = 0
|
||||
for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"):
|
||||
testSpec r, makeTest(name, options, cat)
|
||||
inc testsRun
|
||||
if testsRun == 0:
|
||||
echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run"
|
||||
148
testament/htmlgen.nim
Normal file
148
testament/htmlgen.nim
Normal file
@@ -0,0 +1,148 @@
|
||||
#
|
||||
#
|
||||
# Nim Tester
|
||||
# (c) Copyright 2017 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
## HTML generator for the tester.
|
||||
|
||||
import cgi, backend, strutils, json, os, tables, times
|
||||
|
||||
import "testamenthtml.templ"
|
||||
|
||||
proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) =
|
||||
let
|
||||
trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str).
|
||||
multiReplace({".": "_", " ": "_", ":": "_"})
|
||||
name = testResultRow["name"].str.htmlQuote()
|
||||
category = testResultRow["category"].str.htmlQuote()
|
||||
target = testResultRow["target"].str.htmlQuote()
|
||||
action = testResultRow["action"].str.htmlQuote()
|
||||
result = htmlQuote testResultRow["result"].str
|
||||
expected = testResultRow["expected"].getStr
|
||||
gotten = testResultRow["given"].getStr
|
||||
timestamp = "unknown"
|
||||
var
|
||||
panelCtxClass, textCtxClass, bgCtxClass: string
|
||||
resultSign, resultDescription: string
|
||||
case result
|
||||
of "reSuccess":
|
||||
panelCtxClass = "success"
|
||||
textCtxClass = "success"
|
||||
bgCtxClass = "success"
|
||||
resultSign = "ok"
|
||||
resultDescription = "PASS"
|
||||
of "reIgnored":
|
||||
panelCtxClass = "info"
|
||||
textCtxClass = "info"
|
||||
bgCtxClass = "info"
|
||||
resultSign = "question"
|
||||
resultDescription = "SKIP"
|
||||
else:
|
||||
panelCtxClass = "danger"
|
||||
textCtxClass = "danger"
|
||||
bgCtxClass = "danger"
|
||||
resultSign = "exclamation"
|
||||
resultDescription = "FAIL"
|
||||
|
||||
outfile.generateHtmlTestresultPanelBegin(
|
||||
trId, name, target, category, action, resultDescription,
|
||||
timestamp, result, resultSign, panelCtxClass, textCtxClass, bgCtxClass
|
||||
)
|
||||
if expected.isNilOrWhitespace() and gotten.isNilOrWhitespace():
|
||||
outfile.generateHtmlTestresultOutputNone()
|
||||
else:
|
||||
outfile.generateHtmlTestresultOutputDetails(
|
||||
expected.strip().htmlQuote,
|
||||
gotten.strip().htmlQuote
|
||||
)
|
||||
outfile.generateHtmlTestresultPanelEnd()
|
||||
|
||||
type
|
||||
AllTests = object
|
||||
data: JSonNode
|
||||
totalCount, successCount, ignoredCount, failedCount: int
|
||||
successPercentage, ignoredPercentage, failedPercentage: BiggestFloat
|
||||
|
||||
proc allTestResults(onlyFailing = false): AllTests =
|
||||
result.data = newJArray()
|
||||
for file in os.walkFiles("testresults/*.json"):
|
||||
let data = parseFile(file)
|
||||
if data.kind != JArray:
|
||||
echo "[ERROR] ignoring json file that is not an array: ", file
|
||||
else:
|
||||
for elem in data:
|
||||
let state = elem["result"].str
|
||||
inc result.totalCount
|
||||
if state.contains("reSuccess"): inc result.successCount
|
||||
elif state.contains("reIgnored"): inc result.ignoredCount
|
||||
if not onlyFailing or not(state.contains("reSuccess")):
|
||||
result.data.add elem
|
||||
result.successPercentage = 100 *
|
||||
(result.successCount.toBiggestFloat / result.totalCount.toBiggestFloat)
|
||||
result.ignoredPercentage = 100 *
|
||||
(result.ignoredCount.toBiggestFloat / result.totalCount.toBiggestFloat)
|
||||
result.failedCount = result.totalCount -
|
||||
result.successCount - result.ignoredCount
|
||||
result.failedPercentage = 100 *
|
||||
(result.failedCount.toBiggestFloat / result.totalCount.toBiggestFloat)
|
||||
|
||||
proc generateTestResultsPanelGroupPartial(outfile: File, allResults: JsonNode) =
|
||||
for testresultRow in allResults:
|
||||
generateTestResultPanelPartial(outfile, testresultRow)
|
||||
|
||||
proc generateAllTestsContent(outfile: File, allResults: AllTests,
|
||||
onlyFailing = false) =
|
||||
if allResults.data.len < 1: return # Nothing to do if there is no data.
|
||||
# Only results from one test run means that test run environment info is the
|
||||
# same for all tests
|
||||
let
|
||||
firstRow = allResults.data[0]
|
||||
commit = htmlQuote firstRow["commit"].str
|
||||
branch = htmlQuote firstRow["branch"].str
|
||||
machine = htmlQuote firstRow["machine"].str
|
||||
|
||||
outfile.generateHtmlAllTestsBegin(
|
||||
machine, commit, branch,
|
||||
allResults.totalCount,
|
||||
allResults.successCount,
|
||||
formatBiggestFloat(allResults.successPercentage, ffDecimal, 2) & "%",
|
||||
allResults.ignoredCount,
|
||||
formatBiggestFloat(allResults.ignoredPercentage, ffDecimal, 2) & "%",
|
||||
allResults.failedCount,
|
||||
formatBiggestFloat(allResults.failedPercentage, ffDecimal, 2) & "%",
|
||||
onlyFailing
|
||||
)
|
||||
generateTestResultsPanelGroupPartial(outfile, allResults.data)
|
||||
outfile.generateHtmlAllTestsEnd()
|
||||
|
||||
proc generateHtml*(filename: string, onlyFailing: bool) =
|
||||
let
|
||||
currentTime = getTime().local()
|
||||
timestring = htmlQuote format(currentTime, "yyyy-MM-dd HH:mm:ss 'UTC'zzz")
|
||||
var outfile = open(filename, fmWrite)
|
||||
|
||||
outfile.generateHtmlBegin()
|
||||
|
||||
generateAllTestsContent(outfile, allTestResults(onlyFailing), onlyFailing)
|
||||
|
||||
outfile.generateHtmlEnd(timestring)
|
||||
|
||||
outfile.flushFile()
|
||||
close(outfile)
|
||||
|
||||
proc dumpJsonTestResults*(prettyPrint, onlyFailing: bool) =
|
||||
var
|
||||
outfile = stdout
|
||||
jsonString: string
|
||||
|
||||
let results = allTestResults(onlyFailing)
|
||||
if prettyPrint:
|
||||
jsonString = results.data.pretty()
|
||||
else:
|
||||
jsonString = $ results.data
|
||||
|
||||
outfile.writeLine(jsonString)
|
||||
202
testament/specs.nim
Normal file
202
testament/specs.nim
Normal file
@@ -0,0 +1,202 @@
|
||||
#
|
||||
#
|
||||
# Nim Tester
|
||||
# (c) Copyright 2015 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
import parseutils, strutils, os, osproc, streams, parsecfg
|
||||
|
||||
|
||||
var compilerPrefix* = "compiler" / "nim "
|
||||
|
||||
let isTravis* = existsEnv("TRAVIS")
|
||||
let isAppVeyor* = existsEnv("APPVEYOR")
|
||||
|
||||
proc cmdTemplate*(): string =
|
||||
compilerPrefix & "$target --lib:lib --hints:on -d:testing --nimblePath:tests/deps $options $file"
|
||||
|
||||
type
|
||||
TTestAction* = enum
|
||||
actionCompile = "compile"
|
||||
actionRun = "run"
|
||||
actionReject = "reject"
|
||||
actionRunNoSpec = "runNoSpec"
|
||||
TResultEnum* = enum
|
||||
reNimcCrash, # nim compiler seems to have crashed
|
||||
reMsgsDiffer, # error messages differ
|
||||
reFilesDiffer, # expected and given filenames differ
|
||||
reLinesDiffer, # expected and given line numbers differ
|
||||
reOutputsDiffer,
|
||||
reExitcodesDiffer,
|
||||
reInvalidPeg,
|
||||
reCodegenFailure,
|
||||
reCodeNotFound,
|
||||
reExeNotFound,
|
||||
reInstallFailed # package installation failed
|
||||
reBuildFailed # package building failed
|
||||
reIgnored, # test is ignored
|
||||
reSuccess # test was successful
|
||||
TTarget* = enum
|
||||
targetC = "C"
|
||||
targetCpp = "C++"
|
||||
targetObjC = "ObjC"
|
||||
targetJS = "JS"
|
||||
|
||||
TSpec* = object
|
||||
action*: TTestAction
|
||||
file*, cmd*: string
|
||||
outp*: string
|
||||
line*, column*: int
|
||||
tfile*: string
|
||||
tline*, tcolumn*: int
|
||||
exitCode*: int
|
||||
msg*: string
|
||||
ccodeCheck*: string
|
||||
maxCodeSize*: int
|
||||
err*: TResultEnum
|
||||
substr*, sortoutput*: bool
|
||||
targets*: set[TTarget]
|
||||
nimout*: string
|
||||
|
||||
const
|
||||
targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"]
|
||||
targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"]
|
||||
|
||||
when not declared(parseCfgBool):
|
||||
# candidate for the stdlib:
|
||||
proc parseCfgBool(s: string): bool =
|
||||
case normalize(s)
|
||||
of "y", "yes", "true", "1", "on": result = true
|
||||
of "n", "no", "false", "0", "off": result = false
|
||||
else: raise newException(ValueError, "cannot interpret as a bool: " & s)
|
||||
|
||||
proc extractSpec(filename: string): string =
|
||||
const tripleQuote = "\"\"\""
|
||||
var x = readFile(filename).string
|
||||
var a = x.find(tripleQuote)
|
||||
var b = x.find(tripleQuote, a+3)
|
||||
# look for """ only in the first section
|
||||
if a >= 0 and b > a and a < 40:
|
||||
result = x.substr(a+3, b-1).replace("'''", tripleQuote)
|
||||
else:
|
||||
#echo "warning: file does not contain spec: " & filename
|
||||
result = ""
|
||||
|
||||
when not defined(nimhygiene):
|
||||
{.pragma: inject.}
|
||||
|
||||
template parseSpecAux(fillResult: untyped) =
|
||||
var ss = newStringStream(extractSpec(filename))
|
||||
var p {.inject.}: CfgParser
|
||||
open(p, ss, filename, 1)
|
||||
while true:
|
||||
var e {.inject.} = next(p)
|
||||
case e.kind
|
||||
of cfgEof: break
|
||||
of cfgSectionStart, cfgOption, cfgError:
|
||||
echo ignoreMsg(p, e)
|
||||
of cfgKeyValuePair:
|
||||
fillResult
|
||||
close(p)
|
||||
|
||||
proc specDefaults*(result: var TSpec) =
|
||||
result.msg = ""
|
||||
result.outp = ""
|
||||
result.nimout = ""
|
||||
result.ccodeCheck = ""
|
||||
result.cmd = cmdTemplate()
|
||||
result.line = 0
|
||||
result.column = 0
|
||||
result.tfile = ""
|
||||
result.tline = 0
|
||||
result.tcolumn = 0
|
||||
result.maxCodeSize = 0
|
||||
|
||||
proc parseTargets*(value: string): set[TTarget] =
|
||||
for v in value.normalize.splitWhitespace:
|
||||
case v
|
||||
of "c": result.incl(targetC)
|
||||
of "cpp", "c++": result.incl(targetCpp)
|
||||
of "objc": result.incl(targetObjC)
|
||||
of "js": result.incl(targetJS)
|
||||
else: echo "target ignored: " & v
|
||||
|
||||
proc parseSpec*(filename: string): TSpec =
|
||||
specDefaults(result)
|
||||
result.file = filename
|
||||
parseSpecAux:
|
||||
case normalize(e.key)
|
||||
of "action":
|
||||
case e.value.normalize
|
||||
of "compile": result.action = actionCompile
|
||||
of "run": result.action = actionRun
|
||||
of "reject": result.action = actionReject
|
||||
else: echo ignoreMsg(p, e)
|
||||
of "file": result.file = e.value
|
||||
of "line": discard parseInt(e.value, result.line)
|
||||
of "column": discard parseInt(e.value, result.column)
|
||||
of "tfile": result.tfile = e.value
|
||||
of "tline": discard parseInt(e.value, result.tline)
|
||||
of "tcolumn": discard parseInt(e.value, result.tcolumn)
|
||||
of "output":
|
||||
result.action = actionRun
|
||||
result.outp = e.value
|
||||
of "outputsub":
|
||||
result.action = actionRun
|
||||
result.outp = e.value
|
||||
result.substr = true
|
||||
of "sortoutput":
|
||||
result.sortoutput = parseCfgBool(e.value)
|
||||
of "exitcode":
|
||||
discard parseInt(e.value, result.exitCode)
|
||||
result.action = actionRun
|
||||
of "msg":
|
||||
result.msg = e.value
|
||||
if result.action != actionRun:
|
||||
result.action = actionCompile
|
||||
of "errormsg", "errmsg":
|
||||
result.msg = e.value
|
||||
result.action = actionReject
|
||||
of "nimout":
|
||||
result.nimout = e.value
|
||||
of "disabled":
|
||||
case e.value.normalize
|
||||
of "y", "yes", "true", "1", "on": result.err = reIgnored
|
||||
of "n", "no", "false", "0", "off": discard
|
||||
of "win", "windows":
|
||||
when defined(windows): result.err = reIgnored
|
||||
of "linux":
|
||||
when defined(linux): result.err = reIgnored
|
||||
of "bsd":
|
||||
when defined(bsd): result.err = reIgnored
|
||||
of "macosx":
|
||||
when defined(macosx): result.err = reIgnored
|
||||
of "unix":
|
||||
when defined(unix): result.err = reIgnored
|
||||
of "posix":
|
||||
when defined(posix): result.err = reIgnored
|
||||
of "travis":
|
||||
if isTravis: result.err = reIgnored
|
||||
of "appveyor":
|
||||
if isAppVeyor: result.err = reIgnored
|
||||
else:
|
||||
raise newException(ValueError, "cannot interpret as a bool: " & e.value)
|
||||
of "cmd":
|
||||
if e.value.startsWith("nim "):
|
||||
result.cmd = compilerPrefix & e.value[4..^1]
|
||||
else:
|
||||
result.cmd = e.value
|
||||
of "ccodecheck": result.ccodeCheck = e.value
|
||||
of "maxcodesize": discard parseInt(e.value, result.maxCodeSize)
|
||||
of "target", "targets":
|
||||
for v in e.value.normalize.splitWhitespace:
|
||||
case v
|
||||
of "c": result.targets.incl(targetC)
|
||||
of "cpp", "c++": result.targets.incl(targetCpp)
|
||||
of "objc": result.targets.incl(targetObjC)
|
||||
of "js": result.targets.incl(targetJS)
|
||||
else: echo ignoreMsg(p, e)
|
||||
else: echo ignoreMsg(p, e)
|
||||
297
testament/testamenthtml.templ
Normal file
297
testament/testamenthtml.templ
Normal file
@@ -0,0 +1,297 @@
|
||||
#? stdtmpl(subsChar = '%', metaChar = '#', emit = "outfile.write")
|
||||
#import strutils
|
||||
#
|
||||
#proc htmlQuote*(raw: string): string =
|
||||
# result = raw.multiReplace(
|
||||
# ("&", "&"),
|
||||
# ("\"", """),
|
||||
# ("'", "'"),
|
||||
# ("<", "<"),
|
||||
# (">", ">")
|
||||
# )
|
||||
#
|
||||
#end proc
|
||||
#proc generateHtmlBegin*(outfile: File) =
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Testament Test Results</title>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.0/jquery.min.js" integrity="sha256-ihAoc6M/JPfrIiIeayPE9xjin4UWjsx2mjW/rtmxLM4=" crossorigin="anonymous"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha256-916EbMg70RQy9LHiGkXzG8hSg9EdNy97GazNG/aiY1w=" crossorigin="anonymous" />
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha256-ZT4HPpdCOt2lvDkXokHuhJfdOKSPFLzeAJik5U/Q+l4=" crossorigin="anonymous" />
|
||||
<script>
|
||||
/**
|
||||
* Callback function that is executed for each Element in an array.
|
||||
* @callback executeForElement
|
||||
* @param {Element} elem Element to operate on
|
||||
*/
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {number} index
|
||||
* @param {Element[]} elemArray
|
||||
* @param {executeForElement} executeOnItem
|
||||
*/
|
||||
function executeAllAsync(elemArray, index, executeOnItem) {
|
||||
for (var i = 0; index < elemArray.length && i < 100; i++ , index++) {
|
||||
var item = elemArray[index];
|
||||
executeOnItem(item);
|
||||
}
|
||||
if (index < elemArray.length) {
|
||||
setTimeout(executeAllAsync, 0, elemArray, index, executeOnItem);
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {Element} elem */
|
||||
function executeShowOnElement(elem) {
|
||||
while (elem.classList.contains("hidden")) {
|
||||
elem.classList.remove("hidden");
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {Element} elem */
|
||||
function executeHideOnElement(elem) {
|
||||
if (!elem.classList.contains("hidden")) {
|
||||
elem.classList.add("hidden");
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {Element} elem */
|
||||
function executeExpandOnElement(elem) {
|
||||
$(elem).collapse("show");
|
||||
}
|
||||
|
||||
/** @param {Element} elem */
|
||||
function executeCollapseOnElement(elem) {
|
||||
$(elem).collapse("hide");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
|
||||
* @param {executeForElement} executeOnEachPanel
|
||||
*/
|
||||
function wholePanelAll(category, executeOnEachPanel) {
|
||||
var selector = "div.panel";
|
||||
if (typeof category === "string" && category) {
|
||||
selector += "-" + category;
|
||||
}
|
||||
|
||||
var jqPanels = $(selector);
|
||||
/** @type {Element[]} */
|
||||
var elemArray = jqPanels.toArray();
|
||||
|
||||
setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanel);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
|
||||
* @param {executeForElement} executeOnEachPanel
|
||||
*/
|
||||
function panelBodyAll(category, executeOnEachPanelBody) {
|
||||
var selector = "div.panel";
|
||||
if (typeof category === "string" && category) {
|
||||
selector += "-" + category;
|
||||
}
|
||||
|
||||
var jqPanels = $(selector);
|
||||
|
||||
var jqPanelBodies = $("div.panel-body", jqPanels);
|
||||
/** @type {Element[]} */
|
||||
var elemArray = jqPanelBodies.toArray();
|
||||
|
||||
setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanelBody);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
|
||||
*/
|
||||
function showAll(category) {
|
||||
wholePanelAll(category, executeShowOnElement);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
|
||||
*/
|
||||
function hideAll(category) {
|
||||
wholePanelAll(category, executeHideOnElement);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
|
||||
*/
|
||||
function expandAll(category) {
|
||||
panelBodyAll(category, executeExpandOnElement);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
|
||||
*/
|
||||
function collapseAll(category) {
|
||||
panelBodyAll(category, executeCollapseOnElement);
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>Testament Test Results <small>Nim Tester</small></h1>
|
||||
#end proc
|
||||
#proc generateHtmlAllTestsBegin*(outfile: File, machine, commit, branch: string,
|
||||
# totalCount: BiggestInt,
|
||||
# successCount: BiggestInt, successPercentage: string,
|
||||
# ignoredCount: BiggestInt, ignoredPercentage: string,
|
||||
# failedCount: BiggestInt, failedPercentage: string, onlyFailing = false) =
|
||||
<dl class="dl-horizontal">
|
||||
<dt>Hostname</dt>
|
||||
<dd>%machine</dd>
|
||||
<dt>Git Commit</dt>
|
||||
<dd><code>%commit</code></dd>
|
||||
<dt title="Git Branch reference">Branch ref.</dt>
|
||||
<dd>%branch</dd>
|
||||
</dl>
|
||||
<dl class="dl-horizontal">
|
||||
<dt>All Tests</dt>
|
||||
<dd>
|
||||
<span class="glyphicon glyphicon-th-list"></span>
|
||||
%totalCount
|
||||
</dd>
|
||||
<dt>Successful Tests</dt>
|
||||
<dd>
|
||||
<span class="glyphicon glyphicon-ok-sign"></span>
|
||||
%successCount (%successPercentage)
|
||||
</dd>
|
||||
<dt>Skipped Tests</dt>
|
||||
<dd>
|
||||
<span class="glyphicon glyphicon-question-sign"></span>
|
||||
%ignoredCount (%ignoredPercentage)
|
||||
</dd>
|
||||
<dt>Failed Tests</dt>
|
||||
<dd>
|
||||
<span class="glyphicon glyphicon-exclamation-sign"></span>
|
||||
%failedCount (%failedPercentage)
|
||||
</dd>
|
||||
</dl>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-condensed">
|
||||
# if not onlyFailing:
|
||||
<tr>
|
||||
<th class="text-right" style="vertical-align:middle">All Tests</th>
|
||||
<td>
|
||||
<div class="btn-group">
|
||||
<button class="btn btn-default" type="button" onclick="showAll();">Show All</button>
|
||||
<button class="btn btn-default" type="button" onclick="hideAll();">Hide All</button>
|
||||
<button class="btn btn-default" type="button" onclick="expandAll();">Expand All</button>
|
||||
<button class="btn btn-default" type="button" onclick="collapseAll();">Collapse All</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th class="text-right" style="vertical-align:middle">Successful Tests</th>
|
||||
<td>
|
||||
<div class="btn-group">
|
||||
<button class="btn btn-default" type="button" onclick="showAll('success');">Show All</button>
|
||||
<button class="btn btn-default" type="button" onclick="hideAll('success');">Hide All</button>
|
||||
<button class="btn btn-default" type="button" onclick="expandAll('success');">Expand All</button>
|
||||
<button class="btn btn-default" type="button" onclick="collapseAll('success');">Collapse All</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
# end if
|
||||
<tr>
|
||||
<th class="text-right" style="vertical-align:middle">Skipped Tests</th>
|
||||
<td>
|
||||
<div class="btn-group">
|
||||
<button class="btn btn-default" type="button" onclick="showAll('info');">Show All</button>
|
||||
<button class="btn btn-default" type="button" onclick="hideAll('info');">Hide All</button>
|
||||
<button class="btn btn-default" type="button" onclick="expandAll('info');">Expand All</button>
|
||||
<button class="btn btn-default" type="button" onclick="collapseAll('info');">Collapse All</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th class="text-right" style="vertical-align:middle">Failed Tests</th>
|
||||
<td>
|
||||
<div class="btn-group">
|
||||
<button class="btn btn-default" type="button" onclick="showAll('danger');">Show All</button>
|
||||
<button class="btn btn-default" type="button" onclick="hideAll('danger');">Hide All</button>
|
||||
<button class="btn btn-default" type="button" onclick="expandAll('danger');">Expand All</button>
|
||||
<button class="btn btn-default" type="button" onclick="collapseAll('danger');">Collapse All</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
<div class="panel-group">
|
||||
#end proc
|
||||
#proc generateHtmlTestresultPanelBegin*(outfile: File, trId, name, target, category,
|
||||
# action, resultDescription, timestamp, result, resultSign,
|
||||
# panelCtxClass, textCtxClass, bgCtxClass: string) =
|
||||
<div id="panel-testResult-%trId" class="panel panel-%panelCtxClass">
|
||||
<div class="panel-heading" style="cursor:pointer" data-toggle="collapse" data-target="#panel-body-testResult-%trId" aria-controls="panel-body-testResult-%trId" aria-expanded="false">
|
||||
<div class="row">
|
||||
<h4 class="col-xs-3 col-sm-1 panel-title">
|
||||
<span class="glyphicon glyphicon-%resultSign-sign"></span>
|
||||
<strong>%resultDescription</strong>
|
||||
</h4>
|
||||
<h4 class="col-xs-1 panel-title"><span class="badge">%target</span></h4>
|
||||
<h4 class="col-xs-5 col-sm-7 panel-title" title="%name"><code class="text-%textCtxClass">%name</code></h4>
|
||||
<h4 class="col-xs-3 col-sm-3 panel-title text-right"><span class="badge">%category</span></h4>
|
||||
</div>
|
||||
</div>
|
||||
<div id="panel-body-testResult-%trId" class="panel-body collapse bg-%bgCtxClass">
|
||||
<dl class="dl-horizontal">
|
||||
<dt>Name</dt>
|
||||
<dd><code class="text-%textCtxClass">%name</code></dd>
|
||||
<dt>Category</dt>
|
||||
<dd><span class="badge">%category</span></dd>
|
||||
<dt>Timestamp</dt>
|
||||
<dd>%timestamp</dd>
|
||||
<dt>Nim Action</dt>
|
||||
<dd><code class="text-%textCtxClass">%action</code></dd>
|
||||
<dt>Nim Backend Target</dt>
|
||||
<dd><span class="badge">%target</span></dd>
|
||||
<dt>Code</dt>
|
||||
<dd><code class="text-%textCtxClass">%result</code></dd>
|
||||
</dl>
|
||||
#end proc
|
||||
#proc generateHtmlTestresultOutputDetails*(outfile: File, expected, gotten: string) =
|
||||
<div class="table-responsive">
|
||||
<table class="table table-condensed">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Expected</th>
|
||||
<th>Actual</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><pre>%expected</pre></td>
|
||||
<td><pre>%gotten</pre></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
#end proc
|
||||
#proc generateHtmlTestresultOutputNone*(outfile: File) =
|
||||
<p class="sr-only">No output details</p>
|
||||
#end proc
|
||||
#proc generateHtmlTestresultPanelEnd*(outfile: File) =
|
||||
</div>
|
||||
</div>
|
||||
#end proc
|
||||
#proc generateHtmlAllTestsEnd*(outfile: File) =
|
||||
</div>
|
||||
#end proc
|
||||
#proc generateHtmlEnd*(outfile: File, timestamp: string) =
|
||||
<hr />
|
||||
<footer>
|
||||
<p>
|
||||
Report generated by: <code>testament</code> – Nim Tester
|
||||
<br />
|
||||
Made with Nim. Generated on: %timestamp
|
||||
</p>
|
||||
</footer>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
519
testament/tester.nim
Normal file
519
testament/tester.nim
Normal file
@@ -0,0 +1,519 @@
|
||||
#
|
||||
#
|
||||
# Nim Tester
|
||||
# (c) Copyright 2017 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
## This program verifies Nim against the testcases.
|
||||
|
||||
import
|
||||
parseutils, strutils, pegs, os, osproc, streams, parsecfg, json,
|
||||
marshal, backend, parseopt, specs, htmlgen, browsers, terminal,
|
||||
algorithm, compiler/nodejs, times, sets, md5
|
||||
|
||||
const
|
||||
resultsFile = "testresults.html"
|
||||
#jsonFile = "testresults.json" # not used
|
||||
Usage = """Usage:
|
||||
tester [options] command [arguments]
|
||||
|
||||
Command:
|
||||
all run all tests
|
||||
c|cat|category <category> run all the tests of a certain category
|
||||
r|run <test> run single test file
|
||||
html generate $1 from the database
|
||||
Arguments:
|
||||
arguments are passed to the compiler
|
||||
Options:
|
||||
--print also print results to the console
|
||||
--failing only show failing/ignored tests
|
||||
--targets:"c c++ js objc" run tests for specified targets (default: all)
|
||||
--nim:path use a particular nim executable (default: compiler/nim)
|
||||
""" % resultsFile
|
||||
|
||||
type
|
||||
Category = distinct string
|
||||
TResults = object
|
||||
total, passed, skipped: int
|
||||
data: string
|
||||
|
||||
TTest = object
|
||||
name: string
|
||||
cat: Category
|
||||
options: string
|
||||
action: TTestAction
|
||||
startTime: float
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
let
|
||||
pegLineError =
|
||||
peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
|
||||
pegLineTemplate =
|
||||
peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*"
|
||||
pegOtherError = peg"'Error:' \s* {.*}"
|
||||
pegSuccess = peg"'Hint: operation successful'.*"
|
||||
pegOfInterest = pegLineError / pegOtherError
|
||||
|
||||
var targets = {low(TTarget)..high(TTarget)}
|
||||
|
||||
proc normalizeMsg(s: string): string =
|
||||
result = newStringOfCap(s.len+1)
|
||||
for x in splitLines(s):
|
||||
if result.len > 0: result.add '\L'
|
||||
result.add x.strip
|
||||
|
||||
proc getFileDir(filename: string): string =
|
||||
result = filename.splitFile().dir
|
||||
if not result.isAbsolute():
|
||||
result = getCurrentDir() / result
|
||||
|
||||
proc nimcacheDir(filename, options: string, target: TTarget): string =
|
||||
## Give each test a private nimcache dir so they don't clobber each other's.
|
||||
let hashInput = options & $target
|
||||
return "nimcache" / (filename & '_' & hashInput.getMD5)
|
||||
|
||||
proc callCompiler(cmdTemplate, filename, options: string,
|
||||
target: TTarget, extraOptions=""): TSpec =
|
||||
let nimcache = nimcacheDir(filename, options, target)
|
||||
let options = options & " " & ("--nimCache:" & nimcache).quoteShell & extraOptions
|
||||
let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
|
||||
"options", options, "file", filename.quoteShell,
|
||||
"filedir", filename.getFileDir()])
|
||||
var p = startProcess(command=c[0], args=c[1.. ^1],
|
||||
options={poStdErrToStdOut, poUsePath})
|
||||
let outp = p.outputStream
|
||||
var suc = ""
|
||||
var err = ""
|
||||
var tmpl = ""
|
||||
var x = newStringOfCap(120)
|
||||
result.nimout = ""
|
||||
while outp.readLine(x.TaintedString) or running(p):
|
||||
result.nimout.add(x & "\n")
|
||||
if x =~ pegOfInterest:
|
||||
# `err` should contain the last error/warning message
|
||||
err = x
|
||||
elif x =~ pegLineTemplate and err == "":
|
||||
# `tmpl` contains the last template expansion before the error
|
||||
tmpl = x
|
||||
elif x =~ pegSuccess:
|
||||
suc = x
|
||||
close(p)
|
||||
result.msg = ""
|
||||
result.file = ""
|
||||
result.outp = ""
|
||||
result.line = 0
|
||||
result.column = 0
|
||||
result.tfile = ""
|
||||
result.tline = 0
|
||||
result.tcolumn = 0
|
||||
if tmpl =~ pegLineTemplate:
|
||||
result.tfile = extractFilename(matches[0])
|
||||
result.tline = parseInt(matches[1])
|
||||
result.tcolumn = parseInt(matches[2])
|
||||
if err =~ pegLineError:
|
||||
result.file = extractFilename(matches[0])
|
||||
result.line = parseInt(matches[1])
|
||||
result.column = parseInt(matches[2])
|
||||
result.msg = matches[3]
|
||||
elif err =~ pegOtherError:
|
||||
result.msg = matches[0]
|
||||
elif suc =~ pegSuccess:
|
||||
result.err = reSuccess
|
||||
|
||||
proc callCCompiler(cmdTemplate, filename, options: string,
|
||||
target: TTarget): TSpec =
|
||||
let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
|
||||
"options", options, "file", filename.quoteShell,
|
||||
"filedir", filename.getFileDir()])
|
||||
var p = startProcess(command="gcc", args=c[5 .. ^1],
|
||||
options={poStdErrToStdOut, poUsePath})
|
||||
let outp = p.outputStream
|
||||
var x = newStringOfCap(120)
|
||||
result.nimout = ""
|
||||
result.msg = ""
|
||||
result.file = ""
|
||||
result.outp = ""
|
||||
result.line = -1
|
||||
while outp.readLine(x.TaintedString) or running(p):
|
||||
result.nimout.add(x & "\n")
|
||||
close(p)
|
||||
if p.peekExitCode == 0:
|
||||
result.err = reSuccess
|
||||
|
||||
proc initResults: TResults =
|
||||
result.total = 0
|
||||
result.passed = 0
|
||||
result.skipped = 0
|
||||
result.data = ""
|
||||
|
||||
#proc readResults(filename: string): TResults = # not used
|
||||
# result = marshal.to[TResults](readFile(filename).string)
|
||||
|
||||
#proc writeResults(filename: string, r: TResults) = # not used
|
||||
# writeFile(filename, $$r)
|
||||
|
||||
proc `$`(x: TResults): string =
|
||||
result = ("Tests passed: $1 / $3 <br />\n" &
|
||||
"Tests skipped: $2 / $3 <br />\n") %
|
||||
[$x.passed, $x.skipped, $x.total]
|
||||
|
||||
proc addResult(r: var TResults, test: TTest, target: TTarget,
|
||||
expected, given: string, success: TResultEnum) =
|
||||
let name = test.name.extractFilename & " " & $target & test.options
|
||||
let duration = epochTime() - test.startTime
|
||||
let durationStr = duration.formatFloat(ffDecimal, precision = 8)
|
||||
backend.writeTestResult(name = name,
|
||||
category = test.cat.string,
|
||||
target = $target,
|
||||
action = $test.action,
|
||||
result = $success,
|
||||
expected = expected,
|
||||
given = given)
|
||||
r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
|
||||
if success == reSuccess:
|
||||
styledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)"
|
||||
elif success == reIgnored:
|
||||
styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
|
||||
else:
|
||||
styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
|
||||
styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
|
||||
styledEcho styleBright, fgRed, "Failure: ", $success
|
||||
styledEcho fgYellow, "Expected:"
|
||||
styledEcho styleBright, expected, "\n"
|
||||
styledEcho fgYellow, "Gotten:"
|
||||
styledEcho styleBright, given, "\n"
|
||||
|
||||
if existsEnv("APPVEYOR"):
|
||||
let (outcome, msg) =
|
||||
if success == reSuccess:
|
||||
("Passed", "")
|
||||
elif success == reIgnored:
|
||||
("Skipped", "")
|
||||
else:
|
||||
("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
|
||||
var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options,
|
||||
"-Framework", "nim-testament", "-FileName",
|
||||
test.cat.string,
|
||||
"-Outcome", outcome, "-ErrorMessage", msg,
|
||||
"-Duration", $(duration*1000).int],
|
||||
options={poStdErrToStdOut, poUsePath, poParentStreams})
|
||||
discard waitForExit(p)
|
||||
close(p)
|
||||
|
||||
proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
|
||||
if strip(expected.msg) notin strip(given.msg):
|
||||
r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer)
|
||||
elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg:
|
||||
r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer)
|
||||
elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and
|
||||
"internal error:" notin expected.msg:
|
||||
r.addResult(test, target, expected.file, given.file, reFilesDiffer)
|
||||
elif expected.line != given.line and expected.line != 0 or
|
||||
expected.column != given.column and expected.column != 0:
|
||||
r.addResult(test, target, $expected.line & ':' & $expected.column,
|
||||
$given.line & ':' & $given.column,
|
||||
reLinesDiffer)
|
||||
elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and
|
||||
"internal error:" notin expected.msg:
|
||||
r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer)
|
||||
elif expected.tline != given.tline and expected.tline != 0 or
|
||||
expected.tcolumn != given.tcolumn and expected.tcolumn != 0:
|
||||
r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn,
|
||||
$given.tline & ':' & $given.tcolumn,
|
||||
reLinesDiffer)
|
||||
else:
|
||||
r.addResult(test, target, expected.msg, given.msg, reSuccess)
|
||||
inc(r.passed)
|
||||
|
||||
proc generatedFile(test: TTest, target: TTarget): string =
|
||||
let (_, name, _) = test.name.splitFile
|
||||
let ext = targetToExt[target]
|
||||
result = nimcacheDir(test.name, test.options, target) /
|
||||
(if target == targetJS: "" else: "compiler_") &
|
||||
name.changeFileExt(ext)
|
||||
|
||||
proc needsCodegenCheck(spec: TSpec): bool =
|
||||
result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
|
||||
|
||||
proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string,
|
||||
given: var TSpec) =
|
||||
try:
|
||||
let genFile = generatedFile(test, target)
|
||||
let contents = readFile(genFile).string
|
||||
let check = spec.ccodeCheck
|
||||
if check.len > 0:
|
||||
if check[0] == '\\':
|
||||
# little hack to get 'match' support:
|
||||
if not contents.match(check.peg):
|
||||
given.err = reCodegenFailure
|
||||
elif contents.find(check.peg) < 0:
|
||||
given.err = reCodegenFailure
|
||||
expectedMsg = check
|
||||
if spec.maxCodeSize > 0 and contents.len > spec.maxCodeSize:
|
||||
given.err = reCodegenFailure
|
||||
given.msg = "generated code size: " & $contents.len
|
||||
expectedMsg = "max allowed size: " & $spec.maxCodeSize
|
||||
except ValueError:
|
||||
given.err = reInvalidPeg
|
||||
echo getCurrentExceptionMsg()
|
||||
except IOError:
|
||||
given.err = reCodeNotFound
|
||||
echo getCurrentExceptionMsg()
|
||||
|
||||
proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
|
||||
let exp = expectedNimout.strip.replace("\C\L", "\L")
|
||||
let giv = given.nimout.strip.replace("\C\L", "\L")
|
||||
if exp notin giv:
|
||||
given.err = reMsgsDiffer
|
||||
|
||||
proc makeDeterministic(s: string): string =
|
||||
var x = splitLines(s)
|
||||
sort(x, system.cmp)
|
||||
result = join(x, "\n")
|
||||
|
||||
proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
|
||||
expected: TSpec; r: var TResults) =
|
||||
var expectedmsg: string = ""
|
||||
var givenmsg: string = ""
|
||||
if given.err == reSuccess:
|
||||
if expected.needsCodegenCheck:
|
||||
codegenCheck(test, target, expected, expectedmsg, given)
|
||||
givenmsg = given.msg
|
||||
if expected.nimout.len > 0:
|
||||
expectedmsg = expected.nimout
|
||||
givenmsg = given.nimout.strip
|
||||
nimoutCheck(test, expectedmsg, given)
|
||||
else:
|
||||
givenmsg = given.nimout.strip
|
||||
if given.err == reSuccess: inc(r.passed)
|
||||
r.addResult(test, target, expectedmsg, givenmsg, given.err)
|
||||
|
||||
proc testSpec(r: var TResults, test: TTest, target = targetC) =
|
||||
let tname = test.name.addFileExt(".nim")
|
||||
#echo "TESTING ", tname
|
||||
var expected: TSpec
|
||||
if test.action != actionRunNoSpec:
|
||||
expected = parseSpec(tname)
|
||||
if test.action == actionRun and expected.action == actionCompile:
|
||||
expected.action = actionRun
|
||||
else:
|
||||
specDefaults expected
|
||||
expected.action = actionRunNoSpec
|
||||
|
||||
if expected.err == reIgnored:
|
||||
r.addResult(test, target, "", "", reIgnored)
|
||||
inc(r.skipped)
|
||||
inc(r.total)
|
||||
return
|
||||
|
||||
if expected.targets == {}:
|
||||
expected.targets.incl(target)
|
||||
|
||||
for target in expected.targets:
|
||||
inc(r.total)
|
||||
if target notin targets:
|
||||
r.addResult(test, target, "", "", reIgnored)
|
||||
inc(r.skipped)
|
||||
continue
|
||||
|
||||
case expected.action
|
||||
of actionCompile:
|
||||
var given = callCompiler(expected.cmd, test.name, test.options, target,
|
||||
extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off")
|
||||
compilerOutputTests(test, target, given, expected, r)
|
||||
of actionRun, actionRunNoSpec:
|
||||
# In this branch of code "early return" pattern is clearer than deep
|
||||
# nested conditionals - the empty rows in between to clarify the "danger"
|
||||
var given = callCompiler(expected.cmd, test.name, test.options,
|
||||
target)
|
||||
|
||||
if given.err != reSuccess:
|
||||
r.addResult(test, target, "", given.msg, given.err)
|
||||
continue
|
||||
|
||||
let isJsTarget = target == targetJS
|
||||
var exeFile: string
|
||||
if isJsTarget:
|
||||
let (_, file, _) = splitFile(tname)
|
||||
exeFile = nimcacheDir(test.name, test.options, target) / file & ".js"
|
||||
else:
|
||||
exeFile = changeFileExt(tname, ExeExt)
|
||||
|
||||
if not existsFile(exeFile):
|
||||
r.addResult(test, target, expected.outp, "executable not found", reExeNotFound)
|
||||
continue
|
||||
|
||||
let nodejs = if isJsTarget: findNodeJs() else: ""
|
||||
if isJsTarget and nodejs == "":
|
||||
r.addResult(test, target, expected.outp, "nodejs binary not in PATH",
|
||||
reExeNotFound)
|
||||
continue
|
||||
|
||||
let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile
|
||||
var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut})
|
||||
|
||||
# Treat all failure codes from nodejs as 1. Older versions of nodejs used
|
||||
# to return other codes, but for us it is sufficient to know that it's not 0.
|
||||
if exitCode != 0: exitCode = 1
|
||||
|
||||
let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string))
|
||||
else: strip(buf.string)
|
||||
let expectedOut = strip(expected.outp)
|
||||
|
||||
if exitCode != expected.exitCode:
|
||||
r.addResult(test, target, "exitcode: " & $expected.exitCode,
|
||||
"exitcode: " & $exitCode & "\n\nOutput:\n" &
|
||||
bufB, reExitCodesDiffer)
|
||||
continue
|
||||
|
||||
if bufB != expectedOut and expected.action != actionRunNoSpec:
|
||||
if not (expected.substr and expectedOut in bufB):
|
||||
given.err = reOutputsDiffer
|
||||
r.addResult(test, target, expected.outp, bufB, reOutputsDiffer)
|
||||
continue
|
||||
|
||||
compilerOutputTests(test, target, given, expected, r)
|
||||
continue
|
||||
|
||||
of actionReject:
|
||||
var given = callCompiler(expected.cmd, test.name, test.options,
|
||||
target)
|
||||
cmpMsgs(r, expected, given, test, target)
|
||||
continue
|
||||
|
||||
proc testNoSpec(r: var TResults, test: TTest, target = targetC) =
|
||||
# does not extract the spec because the file is not supposed to have any
|
||||
#let tname = test.name.addFileExt(".nim")
|
||||
inc(r.total)
|
||||
let given = callCompiler(cmdTemplate(), test.name, test.options, target)
|
||||
r.addResult(test, target, "", given.msg, given.err)
|
||||
if given.err == reSuccess: inc(r.passed)
|
||||
|
||||
proc testC(r: var TResults, test: TTest) =
|
||||
# runs C code. Doesn't support any specs, just goes by exit code.
|
||||
let tname = test.name.addFileExt(".c")
|
||||
inc(r.total)
|
||||
styledEcho "Processing ", fgCyan, extractFilename(tname)
|
||||
var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
|
||||
if given.err != reSuccess:
|
||||
r.addResult(test, targetC, "", given.msg, given.err)
|
||||
elif test.action == actionRun:
|
||||
let exeFile = changeFileExt(test.name, ExeExt)
|
||||
var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
|
||||
if exitCode != 0: given.err = reExitCodesDiffer
|
||||
if given.err == reSuccess: inc(r.passed)
|
||||
|
||||
proc testExec(r: var TResults, test: TTest) =
|
||||
# runs executable or script, just goes by exit code
|
||||
inc(r.total)
|
||||
let (outp, errC) = execCmdEx(test.options.strip())
|
||||
var given: TSpec
|
||||
specDefaults(given)
|
||||
if errC == 0:
|
||||
given.err = reSuccess
|
||||
else:
|
||||
given.err = reExitCodesDiffer
|
||||
given.msg = outp.string
|
||||
|
||||
if given.err == reSuccess: inc(r.passed)
|
||||
r.addResult(test, targetC, "", given.msg, given.err)
|
||||
|
||||
proc makeTest(test, options: string, cat: Category, action = actionCompile,
|
||||
env: string = ""): TTest =
|
||||
# start with 'actionCompile', will be overwritten in the spec:
|
||||
result = TTest(cat: cat, name: test, options: options,
|
||||
action: action, startTime: epochTime())
|
||||
|
||||
when defined(windows):
|
||||
const
|
||||
# array of modules disabled from compilation test of stdlib.
|
||||
disabledFiles = ["coro.nim", "fsmonitor.nim"]
|
||||
else:
|
||||
const
|
||||
# array of modules disabled from compilation test of stdlib.
|
||||
disabledFiles = ["-"]
|
||||
|
||||
include categories
|
||||
|
||||
# proc runCaasTests(r: var TResults) =
|
||||
# for test, output, status, mode in caasTestsRunner():
|
||||
# r.addResult(test, "", output & "-> " & $mode,
|
||||
# if status: reSuccess else: reOutputsDiffer)
|
||||
|
||||
proc main() =
|
||||
os.putenv "NIMTEST_COLOR", "never"
|
||||
os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
|
||||
|
||||
backend.open()
|
||||
var optPrintResults = false
|
||||
var optFailing = false
|
||||
|
||||
var targetsStr = ""
|
||||
|
||||
var p = initOptParser()
|
||||
p.next()
|
||||
while p.kind == cmdLongoption:
|
||||
case p.key.string.normalize
|
||||
of "print", "verbose": optPrintResults = true
|
||||
of "failing": optFailing = true
|
||||
of "pedantic": discard "now always enabled"
|
||||
of "targets":
|
||||
targetsStr = p.val.string
|
||||
targets = parseTargets(targetsStr)
|
||||
of "nim": compilerPrefix = p.val.string
|
||||
else: quit Usage
|
||||
p.next()
|
||||
if p.kind != cmdArgument: quit Usage
|
||||
var action = p.key.string.normalize
|
||||
p.next()
|
||||
var r = initResults()
|
||||
case action
|
||||
of "all":
|
||||
let testsDir = "tests" & DirSep
|
||||
var myself = quoteShell(findExe("testament" / "tester"))
|
||||
if targetsStr.len > 0:
|
||||
myself &= " " & quoteShell("--targets:" & targetsStr)
|
||||
|
||||
myself &= " " & quoteShell("--nim:" & compilerPrefix)
|
||||
|
||||
var cmds: seq[string] = @[]
|
||||
let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: ""
|
||||
for kind, dir in walkDir(testsDir):
|
||||
assert testsDir.startsWith(testsDir)
|
||||
let cat = dir[testsDir.len .. ^1]
|
||||
if kind == pcDir and cat notin ["testdata", "nimcache"]:
|
||||
cmds.add(myself & " cat " & quoteShell(cat) & rest)
|
||||
for cat in AdditionalCategories:
|
||||
cmds.add(myself & " cat " & quoteShell(cat) & rest)
|
||||
quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams})
|
||||
of "c", "cat", "category":
|
||||
var cat = Category(p.key)
|
||||
p.next
|
||||
processCategory(r, cat, p.cmdLineRest.string)
|
||||
of "r", "run":
|
||||
let (dir, file) = splitPath(p.key.string)
|
||||
let (_, subdir) = splitPath(dir)
|
||||
var cat = Category(subdir)
|
||||
processSingleTest(r, cat, p.cmdLineRest.string, file)
|
||||
of "html":
|
||||
generateHtml(resultsFile, optFailing)
|
||||
else:
|
||||
quit Usage
|
||||
|
||||
if optPrintResults:
|
||||
if action == "html": openDefaultBrowser(resultsFile)
|
||||
else: echo r, r.data
|
||||
backend.close()
|
||||
var failed = r.total - r.passed - r.skipped
|
||||
if failed != 0:
|
||||
echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ",
|
||||
r.skipped, " failed: ", failed
|
||||
quit(QuitFailure)
|
||||
|
||||
if paramCount() == 0:
|
||||
quit Usage
|
||||
main()
|
||||
1
testament/tester.nim.cfg
Normal file
1
testament/tester.nim.cfg
Normal file
@@ -0,0 +1 @@
|
||||
path = "$nim" # For compiler/nodejs
|
||||
Reference in New Issue
Block a user