mirror of
https://github.com/nim-lang/Nim.git
synced 2026-09-02 20:03:45 +00:00
c87926dadf
Grinding a small figdraw-based program under `nim ic` and diffing its output against the classic backend surfaced eight bugs, four of which silently produced a wrong binary rather than an error. Frontend / build graph (`deps.nim`): * Dead `when`-guarded imports were compiled anyway. `when someStrdefine == "x": import y` is `cvUnknown` to the scanner, which conservatively keeps the edge — right for an edge, but it also gave `y` its own `nim m` rule, so a build died on a package the user never installed because they never selected that backend. Track which edges are speculative and drop a speculative subtree that cannot compile; if the guard was in fact live, the discovery fixpoint puts the node back with the honest `cannot open file`. * Deleting a still-imported module went unnoticed: no mtime moves, so nothing re-fires and `nim ic` relinked a stale binary while `nim c` reported `cannot open file`. Report an unresolvable import from a non-speculatively reached module during the graph scan. * Macro-generated imports were discovered once and then forgotten. Discovery only ran after a failure and the graph is re-derived statically every run, so on a warm build the discovered module had no rules at all and editing it changed nothing. Seed the graph from the `.s.deps` sidecars up front. * Config changes invalidated nothing. nifmake decides staleness from file mtimes and never looks at a rule's command line, so `-d:foo=bar` / `--mm:` / `--threads:` regenerated the build file with the new switches and re-fired zero rules. Reify the configuration as a file and make it an input of every rule. * Command-line switches never reached the children: they replay the project's config files, never the driver's argv, so `nim ic --opt:speed` produced a byte-identical debug binary (likewise `--panics`, `--experimental`, `--passC`). Forward the driver's switches, minus the ones that must differ per child. Artifacts and codegen: * A failed `nim m` still wrote its `.s.bif` and cookies, so nifmake saw the rule as satisfied on the next run: `nim ic` then reported success for a program that does not compile, and generated code from error-bearing AST (or hit an internal error in `ccgexprs`). Never persist an artifact when `errorCounter > 0`. * Top-level destructors were never injected. `sfInjectDestructors` lives on the module symbol, which `moduleFromNifFile` rebuilds from scratch, so `genTopLevelStmt` skipped `injectDestructorCalls` entirely: a module-level `block: let h = openHandle()` never ran `=destroy`. Persist the flag as a `(modflags)` record. `injectdestructors` also has to tolerate the `nkReplayAction` entries the loader prepends to `topLevel`. * `nfFirstWrite` / `nfLastRead` were dropped by the serializer. A sym node is written as a bare NIF `SymUse` token, which has nowhere to put node flags, so the frontend's move analysis never reached the backend: EVERY first assignment to a destructor-bearing local compiled as `=sink`, i.e. `=destroy` on still-zeroed memory followed by a copy, and no read was ever a move. Wrap a sym use in `(nflags ...)` when it carries persistent node flags.
984 lines
38 KiB
Nim
984 lines
38 KiB
Nim
#
|
|
#
|
|
# Nim Testament
|
|
# (c) Copyright 2017 Andreas Rumpf
|
|
#
|
|
# See the file "copying.txt", included in this
|
|
# distribution, for details about the copyright.
|
|
#
|
|
|
|
## This program verifies Nim against the testcases.
|
|
|
|
import
|
|
std/[strutils, pegs, os, osproc, streams, json,
|
|
parseopt, browsers, terminal, exitprocs,
|
|
algorithm, times, intsets, macros, tables]
|
|
|
|
import backend, specs, azure, htmlgen
|
|
|
|
from std/sugar import dup
|
|
import compiler/nodejs
|
|
import lib/stdtest/testutils
|
|
from lib/stdtest/specialpaths import splitTestFile
|
|
from std/private/gitutils import diffStrings
|
|
|
|
import ../dist/checksums/src/checksums/md5
|
|
|
|
proc trimUnitSep(x: var string) =
|
|
let L = x.len
|
|
if L > 0 and x[^1] == '\31':
|
|
setLen x, L-1
|
|
|
|
var useColors = true
|
|
var backendLogging = true
|
|
var simulate = false
|
|
var optVerbose = false
|
|
var useMegatest = true
|
|
var valgrindEnabled = true
|
|
var useIc = false
|
|
## `--ic`: compile every C-target test with `nim ic` instead of `nim c`, so the
|
|
## incremental compiler inherits the whole existing corpus (~10k programs with
|
|
## expected output) instead of the handful of tests written for it by hand.
|
|
## Every invariant the `tests/ic` suite checks is IC-against-IC; this is the
|
|
## part that compares IC against the reference backend at scale.
|
|
|
|
proc verboseCmd(cmd: string) =
|
|
if optVerbose:
|
|
echo "executing: ", cmd
|
|
|
|
const
|
|
failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs
|
|
testsDir = "tests" & DirSep
|
|
resultsFile = "testresults.html"
|
|
Usage = """Usage:
|
|
testament [options] command [arguments]
|
|
|
|
Command:
|
|
p|pat|pattern <glob> run all the tests matching the given pattern
|
|
all run all tests in category folders
|
|
c|cat|category <category> run all the tests of a certain category
|
|
r|run <test> run single test file
|
|
html generate $1 from the database
|
|
Arguments:
|
|
arguments are passed to the compiler
|
|
Options:
|
|
--print print results to the console
|
|
--verbose print commands (compiling and running tests)
|
|
--ic compile C-target tests with `nim ic` (incremental)
|
|
--simulate see what tests would be run but don't run them (for debugging)
|
|
--failing only show failing/ignored tests
|
|
--targets:"c cpp js objc" run tests for specified targets (default: c)
|
|
--nim:path use a particular nim executable (default: $$PATH/nim)
|
|
--directory:dir Change to directory dir before reading the tests or doing anything else.
|
|
--colors:on|off Turn messages coloring on|off.
|
|
--backendLogging:on|off Disable or enable backend logging. By default turned on.
|
|
--megatest:on|off Enable or disable megatest. Default is on.
|
|
--valgrind:on|off Enable or disable valgrind support. Default is on.
|
|
--skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored
|
|
|
|
On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API
|
|
provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN.
|
|
|
|
Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables
|
|
tests with remote networking (as in CI).
|
|
""" % resultsFile
|
|
|
|
proc isNimRepoTests(): bool =
|
|
# this logic could either be specific to cwd, or to some file derived from
|
|
# the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose
|
|
# the former since it's simpler and also works with `testament all`.
|
|
let file = "testament"/"testament.nim.cfg"
|
|
result = file.fileExists
|
|
|
|
type
|
|
Category = distinct string
|
|
TResults = object
|
|
total, passed, skipped: int
|
|
data: string
|
|
TTest = object
|
|
name: string
|
|
cat: Category
|
|
options: string
|
|
testArgs: seq[string]
|
|
spec: TSpec
|
|
startTime: float
|
|
debugInfo: string
|
|
|
|
# ----------------------------------------------------------------------------
|
|
|
|
let
|
|
pegLineError =
|
|
peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
|
|
pegOtherError = peg"'Error:' \s* {.*}"
|
|
pegOfInterest = pegLineError / pegOtherError
|
|
|
|
var gTargets = {low(TTarget)..high(TTarget)}
|
|
var targetsSet = false
|
|
|
|
proc isSuccess(input: string): bool =
|
|
# not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs
|
|
# note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we
|
|
# could handle colors, there would be other issues such as handling other flags
|
|
# that may appear in user config (eg: `--filenames`).
|
|
# Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config
|
|
# stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory
|
|
input.startsWith("Hint: ") and input.endsWith("[SuccessX]")
|
|
|
|
proc getFileDir(filename: string): string =
|
|
result = filename.splitFile().dir
|
|
if not result.isAbsolute():
|
|
result = getCurrentDir() / result
|
|
|
|
proc execCmdEx2(command: string, args: openArray[string]; workingDir: string = "", input: string = ""): tuple[
|
|
cmdLine: string,
|
|
output: string,
|
|
exitCode: int] {.tags:
|
|
[ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
|
|
result = ("", "", 0)
|
|
result.cmdLine.add quoteShell(command)
|
|
for arg in args:
|
|
result.cmdLine.add ' '
|
|
result.cmdLine.add quoteShell(arg)
|
|
verboseCmd(result.cmdLine)
|
|
var p = startProcess(command, workingDir = workingDir, args = args,
|
|
options = {poStdErrToStdOut, poUsePath})
|
|
var outp = outputStream(p)
|
|
|
|
# There is no way to provide input for the child process
|
|
# anymore. Closing it will create EOF on stdin instead of eternal
|
|
# blocking.
|
|
let instream = inputStream(p)
|
|
instream.write(input)
|
|
close instream
|
|
|
|
result.exitCode = -1
|
|
var line = newStringOfCap(120)
|
|
while true:
|
|
if outp.readLine(line):
|
|
result.output.add line
|
|
result.output.add '\n'
|
|
else:
|
|
result.exitCode = peekExitCode(p)
|
|
if result.exitCode != -1: break
|
|
close(p)
|
|
|
|
proc nimcacheDir(filename, options: string, target: TTarget,
|
|
extraOptions = ""): string =
|
|
## Give each test a private nimcache dir so they don't clobber each other's.
|
|
## `extraOptions` (a `matrix:` entry) is part of the key: two matrix variants
|
|
## of one file are two different compilations, and sharing a cache between them
|
|
## means each run invalidates what the previous left. Harmless for the classic
|
|
## backend, which caches only object files, but it makes an incremental cache
|
|
## useless — every variant re-sems the world every time.
|
|
let hashInput = options & extraOptions & $target
|
|
result = "nimcache" / (filename & '_' & hashInput.getMD5)
|
|
|
|
const icWarmupSource = """
|
|
# Generated by testament for `--ic`. Compiling this once fills a shared IC cache
|
|
# with `system` and the stdlib modules the test corpus imports most, so each
|
|
# test's own cold build starts from precompiled NIFs instead of re-semming the
|
|
# world. Mirrors nimony's hastur `tools/warmup.nim` + `prefillFromWarmup`.
|
|
import std/[assertions, macros, strutils, tables, os, typetraits, sequtils,
|
|
sugar, math, options, times, json, sets, algorithm, hashes,
|
|
strformat, parseutils, streams, unicode]
|
|
|
|
proc icWarmupAnchor*(): int =
|
|
# Reference a few generic instantiations the corpus leans on so their
|
|
# `.c.nif` artifacts are precompiled too, not just the modules' interfaces.
|
|
var t = initTable[string, int]()
|
|
t["a"] = 1
|
|
var s = @[1, 2, 3]
|
|
s.sort()
|
|
result = s.len + t.len + "x".repeat(2).len
|
|
"""
|
|
|
|
var icWarmupCaches: Table[string, string]
|
|
## Compile-config key -> shared warm IC cache (or "" when unavailable).
|
|
var buildingIcWarmup = false
|
|
|
|
proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string,
|
|
target: TTarget, extraOptions = ""): string =
|
|
var options = target.defaultOptions & ' ' & options
|
|
if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell)
|
|
options.add ' ' & extraOptions
|
|
# we avoid using `parseCmdLine` which is buggy, refs bug #14343
|
|
result = cmdTemplate % ["target", targetToCmd[target],
|
|
"options", options, "file", filename.quoteShell,
|
|
"filedir", filename.getFileDir(), "nim", compilerPrefix]
|
|
if useIc and target in {targetC, targetCpp}:
|
|
# `--ic:on` turns the ordinary compile command into the IC driver, so the
|
|
# verb is left alone: roughly half the corpus overrides the command wholesale
|
|
# (`cmd: "nim c --gc:arc $file"`), which neither goes through `$target` nor
|
|
# picks up `$options`, and such a test now simply gains the switch. Each also
|
|
# gets a private nimcache, which is what makes it incremental at all.
|
|
#
|
|
# Switches must land BEFORE the project file: anything after it is swallowed
|
|
# into `config.arguments`, and a non-empty `arguments` without `--run` is a
|
|
# hard error ("arguments can only be given if the '--run' option is
|
|
# selected").
|
|
var switches = "--ic:on "
|
|
if nimcache.len > 0 and "--nimCache:" notin result and "--nimcache:" notin result:
|
|
switches.add "--nimCache:" & nimcache.quoteShell & " "
|
|
# `rfind`, not `find`: the private nimcache path embeds the test's file name
|
|
# (`nimcache/tests/destructor/tmove.nim_<hash>`), so the FIRST occurrence is
|
|
# inside a switch's value. The project file is the last one.
|
|
let fileArg = filename.quoteShell
|
|
let at = result.rfind(fileArg)
|
|
if at >= 0: result = result[0 ..< at] & switches & result[at .. ^1]
|
|
else: result.add " " & switches
|
|
|
|
proc icWarmupCache(cmdTemplate, filename, options: string, target: TTarget,
|
|
extraOptions: string): string =
|
|
## The shared warm cache for this test's exact compile configuration, built on
|
|
## first use and kept in `nimcache/` across runs. Keyed by the switches AND the
|
|
## test's directory, because both decide what the artifacts contain: the
|
|
## switches through `-d:`/`--mm:` etc., the directory through the `nim.cfg` /
|
|
## `config.nims` it inherits. A cache built under a different configuration
|
|
## would just be invalidated wholesale on first use, which is worse than none.
|
|
if buildingIcWarmup: return ""
|
|
let dir = filename.getFileDir()
|
|
let key = options & extraOptions & $target & dir
|
|
if icWarmupCaches.hasKey(key): return icWarmupCaches[key]
|
|
result = "nimcache" / ("ic_warmup_" & key.getMD5)
|
|
icWarmupCaches[key] = result
|
|
if dirExists(result / "ic.version"): return # already built by an earlier run
|
|
if fileExists(result / "ic.version"): return
|
|
# The warmup must live in the test's own directory so it inherits the same
|
|
# config files; a stray `.nim` there is not picked up as a test (testament
|
|
# only collects `t*.nim`). The name must be a valid Nim identifier.
|
|
let src = dir / "icwarmup_generated.nim"
|
|
try:
|
|
writeFile(src, icWarmupSource)
|
|
except IOError, OSError:
|
|
icWarmupCaches[key] = ""
|
|
return ""
|
|
buildingIcWarmup = true
|
|
let cmd = prepareTestCmd(cmdTemplate, src, options, result, target, extraOptions)
|
|
let (outp, code) = execCmdEx(cmd)
|
|
buildingIcWarmup = false
|
|
try: removeFile(src)
|
|
except OSError: discard
|
|
if code != 0:
|
|
# Non-fatal: without a warm cache every test just pays its own cold build.
|
|
if optVerbose: echo "ic warmup failed: ", cmd, "\n", outp
|
|
icWarmupCaches[key] = ""
|
|
return ""
|
|
|
|
proc prefillIcCache(warmup, nimcache: string) =
|
|
## Seed a test's empty cache from the shared warm one. Only the artifacts that
|
|
## do NOT depend on which program is being built are copied: the frontend NIFs
|
|
## and cookies, plus the per-module `lower`/`cg` outputs. The `.c`/`.o` are
|
|
## deliberately left out — the merge decision (who owns each emit-everywhere
|
|
## definition) is whole-program, so those get re-rendered for every program
|
|
## anyway and copying them is pure I/O.
|
|
##
|
|
## Mtimes are preserved, and that is load-bearing: nifmake decides staleness by
|
|
## output-mtime > input-mtime, so stamping every prefilled file with "now"
|
|
## would scramble the DAG ordering the warmup established and re-fire the
|
|
## whole graph — exactly what the copy is meant to avoid.
|
|
if warmup.len == 0 or not dirExists(warmup): return
|
|
if dirExists(nimcache): return # the test already has its own cache
|
|
const wanted = [".p.nif", ".p.deps.nif", ".deps.nif", ".s.bif", ".iface.bif",
|
|
".impl.bif", ".edges.bif", ".s.deps.bif", ".t.bif",
|
|
".c.nif", ".cpp.nif"]
|
|
try:
|
|
createDir(nimcache)
|
|
for path in walkFiles(warmup / "*"):
|
|
let name = path.extractFilename
|
|
var take = name == "ic.version" or name == "ic_build_args.txt"
|
|
if not take:
|
|
for ext in wanted:
|
|
if name.endsWith(ext): take = true; break
|
|
if not take: continue
|
|
let dst = nimcache / name
|
|
copyFile(path, dst)
|
|
try: setLastModificationTime(dst, getLastModificationTime(path))
|
|
except OSError, IOError: discard
|
|
except OSError, IOError:
|
|
discard # best effort; a cold build still works
|
|
|
|
proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
|
|
target: TTarget, extraOptions = ""): TSpec =
|
|
if useIc and target in {targetC, targetCpp} and nimcache.len > 0 and not buildingIcWarmup:
|
|
prefillIcCache(icWarmupCache(cmdTemplate, filename, options, target, extraOptions),
|
|
nimcache)
|
|
result = TSpec(cmd: prepareTestCmd(cmdTemplate, filename, options, nimcache, target,
|
|
extraOptions))
|
|
verboseCmd(result.cmd)
|
|
var p = startProcess(command = result.cmd,
|
|
options = {poStdErrToStdOut, poUsePath, poEvalCommand})
|
|
let outp = p.outputStream
|
|
var foundSuccessMsg = false
|
|
var foundErrorMsg = false
|
|
var err = ""
|
|
var x = newStringOfCap(120)
|
|
result.nimout = ""
|
|
while true:
|
|
if outp.readLine(x):
|
|
trimUnitSep x
|
|
result.nimout.add(x & '\n')
|
|
if x =~ pegOfInterest:
|
|
# `err` should contain the last error message
|
|
err = x
|
|
foundErrorMsg = true
|
|
elif x.isSuccess:
|
|
foundSuccessMsg = true
|
|
elif not running(p):
|
|
break
|
|
result.msg = ""
|
|
result.file = ""
|
|
result.output = ""
|
|
result.line = 0
|
|
result.column = 0
|
|
|
|
result.err = reNimcCrash
|
|
result.exitCode = p.peekExitCode
|
|
close p
|
|
case result.exitCode
|
|
of 0:
|
|
if foundErrorMsg:
|
|
result.debugInfo.add " compiler exit code was 0 but some Error's were found."
|
|
else:
|
|
result.err = reSuccess
|
|
of 1:
|
|
if not foundErrorMsg:
|
|
result.debugInfo.add " compiler exit code was 1 but no Error's were found."
|
|
if foundSuccessMsg:
|
|
result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true."
|
|
else:
|
|
result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode
|
|
|
|
if err =~ pegLineError:
|
|
result.file = extractFilename(matches[0])
|
|
result.line = parseInt(matches[1])
|
|
result.column = parseInt(matches[2])
|
|
result.msg = matches[3]
|
|
elif err =~ pegOtherError:
|
|
result.msg = matches[0]
|
|
trimUnitSep result.msg
|
|
|
|
proc initResults: TResults =
|
|
result = TResults(
|
|
total: 0,
|
|
passed: 0,
|
|
skipped: 0,
|
|
data: ""
|
|
)
|
|
|
|
macro ignoreStyleEcho(args: varargs[typed]): untyped =
|
|
let typForegroundColor = bindSym"ForegroundColor".getType
|
|
let typBackgroundColor = bindSym"BackgroundColor".getType
|
|
let typStyle = bindSym"Style".getType
|
|
let typTerminalCmd = bindSym"TerminalCmd".getType
|
|
result = newCall(bindSym"echo")
|
|
for arg in children(args):
|
|
if arg.kind == nnkNilLit: continue
|
|
let typ = arg.getType
|
|
if typ.kind != nnkEnumTy or
|
|
typ != typForegroundColor and
|
|
typ != typBackgroundColor and
|
|
typ != typStyle and
|
|
typ != typTerminalCmd:
|
|
result.add(arg)
|
|
|
|
template maybeStyledEcho(args: varargs[untyped]): untyped =
|
|
if useColors:
|
|
styledEcho(args)
|
|
else:
|
|
ignoreStyleEcho(args)
|
|
|
|
|
|
proc `$`(x: TResults): string =
|
|
result = """
|
|
Tests passed: $2 / $1 <br />
|
|
Tests skipped: $3 / $1 <br />
|
|
""" % [$x.total, $x.passed, $x.skipped]
|
|
|
|
proc testName(test: TTest, target: TTarget, extraOptions: string): string =
|
|
var name = test.name.replace(DirSep, '/')
|
|
name.add ' ' & $target
|
|
if test.options.len > 0: name.add ' ' & test.options
|
|
if extraOptions.len > 0: name.add ' ' & extraOptions
|
|
name.strip()
|
|
|
|
proc addResult(r: var TResults, test: TTest, target: TTarget,
|
|
extraOptions, expected, given: string, success: TResultEnum, duration: float,
|
|
givenSpec: ptr TSpec = nil) =
|
|
# instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need
|
|
# instead of having to pass individual fields, or abusing existing ones like expected vs given.
|
|
# test.name is easier to find than test.name.extractFilename
|
|
# A bit hacky but simple and works with tests/testament/tshould_not_work.nim
|
|
let name = testName(test, target, extraOptions)
|
|
|
|
let durationStr = duration.formatFloat(ffDecimal, precision = 2).align(5)
|
|
if backendLogging:
|
|
backend.writeTestResult(name = name,
|
|
category = test.cat.string,
|
|
target = $target,
|
|
action = $test.spec.action,
|
|
result = $success,
|
|
expected = expected,
|
|
given = given)
|
|
r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
|
|
template dispNonSkipped(color, outcome) =
|
|
maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
|
|
template disp(msg) =
|
|
maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name
|
|
if success == reSuccess:
|
|
dispNonSkipped(fgGreen, "PASS: ")
|
|
elif success == reDisabled:
|
|
if test.spec.inCurrentBatch: disp("SKIP:")
|
|
else: disp("NOTINBATCH:")
|
|
elif success == reJoined: disp("JOINED:")
|
|
else:
|
|
dispNonSkipped(fgRed, failString)
|
|
maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
|
|
maybeStyledEcho styleBright, fgRed, "Failure: ", $success
|
|
if givenSpec != nil and givenSpec.debugInfo.len > 0:
|
|
echo "debugInfo: " & givenSpec.debugInfo
|
|
if success in {reBuildFailed, reNimcCrash, reInstallFailed}:
|
|
# expected is empty, no reason to print it.
|
|
echo given
|
|
else:
|
|
maybeStyledEcho fgYellow, "Expected:"
|
|
maybeStyledEcho styleBright, expected, "\n"
|
|
maybeStyledEcho fgYellow, "Gotten:"
|
|
maybeStyledEcho styleBright, given, "\n"
|
|
echo diffStrings(expected, given).output
|
|
|
|
if backendLogging and (isAppVeyor or isAzure):
|
|
let (outcome, msg) =
|
|
case success
|
|
of reSuccess:
|
|
("Passed", "")
|
|
of reDisabled, reJoined:
|
|
("Skipped", "")
|
|
of reBuildFailed, reNimcCrash, reInstallFailed:
|
|
("Failed", "Failure: " & $success & '\n' & given)
|
|
else:
|
|
("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
|
|
if isAzure:
|
|
azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success)
|
|
else:
|
|
var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options,
|
|
"-Framework", "nim-testament", "-FileName",
|
|
test.cat.string,
|
|
"-Outcome", outcome, "-ErrorMessage", msg,
|
|
"-Duration", $(duration * 1000).int],
|
|
options = {poStdErrToStdOut, poUsePath, poParentStreams})
|
|
discard waitForExit(p)
|
|
close(p)
|
|
|
|
proc finishTest(r: var TResults, test: TTest, target: TTarget,
|
|
extraOptions, expected, given: string, successOrig: TResultEnum,
|
|
givenSpec: ptr TSpec = nil) =
|
|
## calculates duration of test, reports result
|
|
## `retries` option in the test is ignored
|
|
let duration = epochTime() - test.startTime
|
|
let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout
|
|
else: successOrig
|
|
addResult(r, test, target, extraOptions, expected, given, success, duration, givenSpec)
|
|
|
|
proc finishTestRetryable(r: var TResults, test: TTest, target: TTarget,
|
|
extraOptions, expected, given: string, successOrig: TResultEnum,
|
|
givenSpec: ptr TSpec = nil): bool =
|
|
## if test failed and has remaining retries, return `true`,
|
|
## otherwise calculate duration and report result
|
|
##
|
|
## warning: if `true` is returned, then the result is not reported,
|
|
## it has to be retried or `finishTest` should be called instead
|
|
result = false
|
|
let duration = epochTime() - test.startTime
|
|
let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout
|
|
else: successOrig
|
|
if test.spec.retries > 0 and success notin {reSuccess, reDisabled, reJoined, reInvalidSpec}:
|
|
return true
|
|
else:
|
|
addResult(r, test, target, extraOptions, expected, given, success, duration, givenSpec)
|
|
|
|
proc toString(inlineError: InlineError, filename: string): string =
|
|
result = "$file($line, $col) $kind: $msg" % [
|
|
"file", filename,
|
|
"line", $inlineError.line,
|
|
"col", $inlineError.col,
|
|
"kind", $inlineError.kind,
|
|
"msg", $inlineError.msg
|
|
]
|
|
|
|
proc inlineErrorsMsgs(expected: TSpec): string =
|
|
result = ""
|
|
for inlineError in expected.inlineErrors.items:
|
|
result.addLine inlineError.toString(expected.filename)
|
|
|
|
proc checkForInlineErrors(expected, given: TSpec): bool =
|
|
for inlineError in expected.inlineErrors:
|
|
if inlineError.toString(expected.filename) notin given.nimout:
|
|
return false
|
|
true
|
|
|
|
proc nimoutCheck(expected, given: TSpec): bool =
|
|
result = true
|
|
if expected.nimoutFull:
|
|
if expected.nimout != given.nimout:
|
|
result = false
|
|
elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
|
|
result = false
|
|
|
|
proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
|
|
target: TTarget, extraOptions: string): bool =
|
|
# result has to be checked for retry
|
|
if not checkForInlineErrors(expected, given) or
|
|
(not expected.nimoutFull and not nimoutCheck(expected, given)):
|
|
result = r.finishTestRetryable(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer)
|
|
elif strip(expected.msg) notin strip(given.msg):
|
|
result = r.finishTestRetryable(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
|
|
elif not nimoutCheck(expected, given):
|
|
result = r.finishTestRetryable(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
|
|
elif extractFilename(expected.file) != extractFilename(given.file) and
|
|
"internal error:" notin expected.msg:
|
|
result = r.finishTestRetryable(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
|
|
elif expected.line != given.line and expected.line != 0 or
|
|
expected.column != given.column and expected.column != 0:
|
|
result = r.finishTestRetryable(test, target, extraOptions, $expected.line & ':' & $expected.column,
|
|
$given.line & ':' & $given.column, reLinesDiffer)
|
|
else:
|
|
result = r.finishTestRetryable(test, target, extraOptions, expected.msg, given.msg, reSuccess)
|
|
inc(r.passed)
|
|
|
|
proc generatedFile(test: TTest, target: TTarget, extraOptions: string): string =
|
|
if target == targetJS:
|
|
result = test.name.changeFileExt("js")
|
|
else:
|
|
let (_, name, _) = test.name.splitFile
|
|
let ext = targetToExt[target]
|
|
# `extraOptions` must match what `testSpecWithNimcache` passed to the
|
|
# compiler — the matrix entry is part of the nimcache key, so leaving it out
|
|
# here looks for the `.c` of a DIFFERENT variant's cache (which does not
|
|
# exist) and every `ccodeCheck` test with a `matrix:` failed as
|
|
# `reCodeNotFound`.
|
|
result = nimcacheDir(test.name, test.options, target, extraOptions) /
|
|
"@m" & name.changeFileExt(ext)
|
|
|
|
proc needsCodegenCheck(spec: TSpec): bool =
|
|
result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
|
|
|
|
proc codegenCheck(test: TTest, target: TTarget, extraOptions: string,
|
|
spec: TSpec, expectedMsg: var string,
|
|
given: var TSpec) =
|
|
try:
|
|
let genFile = generatedFile(test, target, extraOptions)
|
|
let contents = readFile(genFile)
|
|
for check in spec.ccodeCheck:
|
|
if check.len > 0 and check[0] == '\\':
|
|
# little hack to get 'match' support:
|
|
if not contents.match(check.peg):
|
|
given.err = reCodegenFailure
|
|
elif contents.find(check.peg) < 0:
|
|
given.err = reCodegenFailure
|
|
expectedMsg = check
|
|
if spec.maxCodeSize > 0 and contents.len > spec.maxCodeSize:
|
|
given.err = reCodegenFailure
|
|
given.msg = "generated code size: " & $contents.len
|
|
expectedMsg = "max allowed size: " & $spec.maxCodeSize
|
|
except ValueError:
|
|
given.err = reInvalidPeg
|
|
echo getCurrentExceptionMsg()
|
|
except IOError:
|
|
given.err = reCodeNotFound
|
|
echo getCurrentExceptionMsg()
|
|
|
|
proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
|
|
given: var TSpec, expected: TSpec; r: var TResults): bool =
|
|
# result has to be checked for retry
|
|
var expectedmsg: string = ""
|
|
var givenmsg: string = ""
|
|
if given.err == reSuccess:
|
|
if expected.needsCodegenCheck:
|
|
codegenCheck(test, target, extraOptions, expected, expectedmsg, given)
|
|
givenmsg = given.msg
|
|
if not nimoutCheck(expected, given) or
|
|
not checkForInlineErrors(expected, given):
|
|
given.err = reMsgsDiffer
|
|
expectedmsg = expected.nimout & inlineErrorsMsgs(expected)
|
|
givenmsg = given.nimout.strip
|
|
else:
|
|
givenmsg = "$ " & given.cmd & '\n' & given.nimout
|
|
if given.err == reSuccess: inc(r.passed)
|
|
result = r.finishTestRetryable(test, target, extraOptions, expectedmsg, givenmsg, given.err)
|
|
|
|
proc getTestSpecTarget(): TTarget =
|
|
if getEnv("NIM_COMPILE_TO_CPP", "false") == "true":
|
|
result = targetCpp
|
|
else:
|
|
result = targetC
|
|
|
|
var count = 0
|
|
|
|
proc equalModuloLastNewline(a, b: string): bool =
|
|
# allow lazy output spec that omits last newline, but really those should be fixed instead
|
|
result = a == b or b.endsWith("\n") and a == b[0 ..< ^1]
|
|
|
|
proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
|
|
target: TTarget, extraOptions: string, nimcache: string) =
|
|
template maybeRetry(x: bool) =
|
|
# if `x` is true, retries the test
|
|
if x:
|
|
test.spec.err = reRetry
|
|
dec test.spec.retries
|
|
testSpecHelper(r, test, expected, target, extraOptions, nimcache)
|
|
return
|
|
if test.spec.err != reRetry:
|
|
test.startTime = epochTime()
|
|
if testName(test, target, extraOptions) in skips:
|
|
test.spec.err = reDisabled
|
|
|
|
if test.spec.err in {reDisabled, reJoined}:
|
|
r.finishTest(test, target, extraOptions, "", "", test.spec.err)
|
|
inc(r.skipped)
|
|
return
|
|
var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
|
|
case expected.action
|
|
of actionCompile:
|
|
maybeRetry compilerOutputTests(test, target, extraOptions, given, expected, r)
|
|
of actionRun:
|
|
if given.err != reSuccess:
|
|
maybeRetry r.finishTestRetryable(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
|
|
else:
|
|
let isJsTarget = target == targetJS
|
|
var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt)
|
|
if not fileExists(exeFile):
|
|
maybeRetry r.finishTestRetryable(test, target, extraOptions, expected.output,
|
|
"executable not found: " & exeFile, reExeNotFound)
|
|
else:
|
|
let nodejs = if isJsTarget: findNodeJs() else: ""
|
|
if isJsTarget and nodejs == "":
|
|
maybeRetry r.finishTestRetryable(test, target, extraOptions, expected.output, "nodejs binary not in PATH",
|
|
reExeNotFound)
|
|
else:
|
|
var exeCmd: string
|
|
var args = test.testArgs
|
|
if isJsTarget:
|
|
exeCmd = nodejs
|
|
# see D20210217T215950
|
|
args = @["--unhandled-rejections=strict", exeFile] & args
|
|
else:
|
|
exeCmd = exeFile.dup(normalizeExe)
|
|
if valgrindEnabled and expected.useValgrind != disabled:
|
|
var valgrindOptions = @["--error-exitcode=1"]
|
|
if expected.useValgrind != leaking:
|
|
valgrindOptions.add "--leak-check=yes"
|
|
args = valgrindOptions & exeCmd & args
|
|
exeCmd = "valgrind"
|
|
var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input)
|
|
# Treat all failure codes from nodejs as 1. Older versions of nodejs used
|
|
# to return other codes, but for us it is sufficient to know that it's not 0.
|
|
if exitCode != 0: exitCode = 1
|
|
let bufB =
|
|
if expected.sortoutput:
|
|
var buf2 = buf
|
|
buf2.stripLineEnd
|
|
var x = splitLines(buf2)
|
|
sort(x, system.cmp)
|
|
join(x, "\n") & '\n'
|
|
else:
|
|
buf
|
|
if exitCode != expected.exitCode:
|
|
given.err = reExitcodesDiffer
|
|
maybeRetry r.finishTestRetryable(test, target, extraOptions, "exitcode: " & $expected.exitCode,
|
|
"exitcode: " & $exitCode & "\n\nOutput:\n" &
|
|
bufB, reExitcodesDiffer)
|
|
elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or
|
|
(expected.outputCheck == ocSubstr and expected.output notin bufB):
|
|
given.err = reOutputsDiffer
|
|
maybeRetry r.finishTestRetryable(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
|
|
maybeRetry compilerOutputTests(test, target, extraOptions, given, expected, r)
|
|
of actionReject:
|
|
# Make sure its the compiler rejecting and not the system (e.g. segfault)
|
|
maybeRetry cmpMsgs(r, expected, given, test, target, extraOptions)
|
|
if given.exitCode != QuitFailure:
|
|
maybeRetry r.finishTestRetryable(test, target, extraOptions, "exitcode: " & $QuitFailure,
|
|
"exitcode: " & $given.exitCode & "\n\nOutput:\n" &
|
|
given.nimout, reExitcodesDiffer)
|
|
|
|
|
|
|
|
proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget =
|
|
result = defaultTarget
|
|
var p = parseopt.initOptParser(extraOptions)
|
|
|
|
while true:
|
|
parseopt.next(p)
|
|
case p.kind
|
|
of cmdEnd: break
|
|
of cmdLongOption, cmdShortOption:
|
|
if p.key == "b" or p.key == "backend":
|
|
result = parseEnum[TTarget](p.val.normalize)
|
|
# chooses the last one
|
|
else:
|
|
discard
|
|
|
|
proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =
|
|
for target in expected.targets:
|
|
inc(r.total)
|
|
if target notin gTargets:
|
|
r.finishTest(test, target, extraOptions, "", "", reDisabled)
|
|
inc(r.skipped)
|
|
elif simulate:
|
|
inc count
|
|
echo "testSpec count: ", count, " expected: ", expected
|
|
else:
|
|
let nimcache = nimcacheDir(test.name, test.options, target, extraOptions)
|
|
var testClone = test
|
|
let target = changeTarget(extraOptions, target)
|
|
testSpecHelper(r, testClone, expected, target, extraOptions, nimcache)
|
|
|
|
proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
|
|
var expected = test.spec
|
|
if expected.parseErrors.len > 0:
|
|
# targetC is a lie, but a parameter is required
|
|
r.finishTest(test, targetC, "", "", expected.parseErrors, reInvalidSpec)
|
|
inc(r.total)
|
|
return
|
|
|
|
expected.targets.incl targets
|
|
# still no target specified at all
|
|
if expected.targets == {}:
|
|
expected.targets = {getTestSpecTarget()}
|
|
if test.spec.matrix.len > 0:
|
|
for m in test.spec.matrix:
|
|
targetHelper(r, test, expected, m)
|
|
else:
|
|
targetHelper(r, test, expected, "")
|
|
|
|
proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} =
|
|
for target in test.spec.targets:
|
|
inc(r.total)
|
|
var testClone = test
|
|
testSpecHelper(r, testClone, test.spec, target, "", nimcache)
|
|
|
|
proc makeTest(test, options: string, cat: Category): TTest =
|
|
result = TTest(
|
|
cat: cat,
|
|
name: test,
|
|
options: options,
|
|
spec: parseSpec(addFileExt(test, ".nim")),
|
|
startTime: epochTime()
|
|
)
|
|
|
|
proc makeRawTest(test, options: string, cat: Category): TTest {.used.} =
|
|
result = TTest(cat: cat, name: test, options: options,
|
|
spec: initSpec(addFileExt(test, ".nim"))
|
|
)
|
|
result.spec.action = actionCompile
|
|
result.spec.targets = {getTestSpecTarget()}
|
|
result.startTime = epochTime()
|
|
|
|
# TODO: fix these files
|
|
const disabledFilesDefault = @[
|
|
"tableimpl.nim",
|
|
"setimpl.nim",
|
|
"hashcommon.nim",
|
|
|
|
# Requires compiling with '--threads:on`
|
|
"sharedlist.nim",
|
|
"sharedtables.nim",
|
|
|
|
# Error: undeclared identifier: 'hasThreadSupport'
|
|
"ioselectors_epoll.nim",
|
|
"ioselectors_kqueue.nim",
|
|
"ioselectors_poll.nim",
|
|
|
|
# Error: undeclared identifier: 'Timeval'
|
|
"ioselectors_select.nim",
|
|
]
|
|
|
|
when defined(windows):
|
|
const
|
|
# array of modules disabled from compilation test of stdlib.
|
|
disabledFiles = disabledFilesDefault & @["coro.nim"]
|
|
else:
|
|
const
|
|
# array of modules disabled from compilation test of stdlib.
|
|
disabledFiles = disabledFilesDefault
|
|
|
|
include categories
|
|
|
|
proc loadSkipFrom(name: string): seq[string] =
|
|
result = @[]
|
|
if name.len == 0: return
|
|
# One skip per line, comments start with #
|
|
# used by `nlvm` (at least)
|
|
for line in lines(name):
|
|
let sline = line.strip()
|
|
if sline.len > 0 and not sline.startsWith('#'):
|
|
result.add sline
|
|
|
|
proc main() =
|
|
azure.init()
|
|
backend.open()
|
|
var optPrintResults = false
|
|
var optFailing = false
|
|
var targetsStr = ""
|
|
var isMainProcess = true
|
|
var skipFrom = ""
|
|
|
|
var p = initOptParser()
|
|
p.next()
|
|
while p.kind in {cmdLongOption, cmdShortOption}:
|
|
case p.key.normalize
|
|
of "print": optPrintResults = true
|
|
of "verbose": optVerbose = true
|
|
of "ic": useIc = true
|
|
of "failing": optFailing = true
|
|
of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731
|
|
of "targets":
|
|
targetsStr = p.val
|
|
gTargets = parseTargets(targetsStr)
|
|
targetsSet = true
|
|
of "nim":
|
|
compilerPrefix = addFileExt(p.val.absolutePath, ExeExt)
|
|
of "directory":
|
|
setCurrentDir(p.val)
|
|
of "colors":
|
|
case p.val:
|
|
of "on":
|
|
useColors = true
|
|
of "off":
|
|
useColors = false
|
|
else:
|
|
quit Usage
|
|
of "batch":
|
|
testamentData0.batchArg = p.val
|
|
if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}:
|
|
let s = p.val.split("_")
|
|
doAssert s.len == 2, $(p.val, s)
|
|
testamentData0.testamentBatch = s[0].parseInt
|
|
testamentData0.testamentNumBatch = s[1].parseInt
|
|
doAssert testamentData0.testamentNumBatch > 0
|
|
doAssert testamentData0.testamentBatch >= 0 and testamentData0.testamentBatch < testamentData0.testamentNumBatch
|
|
of "simulate":
|
|
simulate = true
|
|
of "megatest":
|
|
case p.val:
|
|
of "on":
|
|
useMegatest = true
|
|
of "off":
|
|
useMegatest = false
|
|
else:
|
|
quit Usage
|
|
of "valgrind":
|
|
case p.val:
|
|
of "on":
|
|
valgrindEnabled = true
|
|
of "off":
|
|
valgrindEnabled = false
|
|
else:
|
|
quit Usage
|
|
of "backendlogging":
|
|
case p.val:
|
|
of "on":
|
|
backendLogging = true
|
|
of "off":
|
|
backendLogging = false
|
|
else:
|
|
quit Usage
|
|
of "skipfrom":
|
|
skipFrom = p.val
|
|
else:
|
|
quit Usage
|
|
p.next()
|
|
if p.kind != cmdArgument:
|
|
quit Usage
|
|
var action = p.key.normalize
|
|
p.next()
|
|
var r = initResults()
|
|
case action
|
|
of "all":
|
|
#processCategory(r, Category"megatest", p.cmdLineRest, testsDir, runJoinableTests = false)
|
|
|
|
var myself = quoteShell(getAppFilename())
|
|
if targetsStr.len > 0:
|
|
myself &= " " & quoteShell("--targets:" & targetsStr)
|
|
|
|
myself &= " " & quoteShell("--nim:" & compilerPrefix)
|
|
if testamentData0.batchArg.len > 0:
|
|
myself &= " --batch:" & testamentData0.batchArg
|
|
|
|
if skipFrom.len > 0:
|
|
myself &= " " & quoteShell("--skipFrom:" & skipFrom)
|
|
|
|
var cats: seq[string] = @[]
|
|
let rest = if p.cmdLineRest.len > 0: " " & p.cmdLineRest else: ""
|
|
for kind, dir in walkDir(testsDir):
|
|
assert testsDir.startsWith(testsDir)
|
|
let cat = dir[testsDir.len .. ^1]
|
|
if kind == pcDir and cat notin ["testdata", "nimcache"]:
|
|
cats.add cat
|
|
if isNimRepoTests():
|
|
# `ic` and `navigator` are real `tests/` dirs (already collected above) *and*
|
|
# listed in AdditionalCategories; without this guard they'd run twice, which
|
|
# doubled the (expensive) `ic` category on every `all` run. `debugger`,
|
|
# `examples` and `lib` have no matching `tests/` dir, so they are still added.
|
|
for cat in AdditionalCategories:
|
|
if cat notin cats: cats.add cat
|
|
if useMegatest: cats.add MegaTestCat
|
|
|
|
var cmds: seq[string] = @[]
|
|
for cat in cats:
|
|
let runtype = if useMegatest: " pcat " else: " cat "
|
|
cmds.add(myself & runtype & quoteShell(cat) & rest)
|
|
|
|
proc progressStatus(idx: int) =
|
|
echo "progress[all]: $1/$2 starting: cat: $3" % [$idx, $cats.len, cats[idx]]
|
|
|
|
if simulate:
|
|
skips = loadSkipFrom(skipFrom)
|
|
for i, cati in cats:
|
|
progressStatus(i)
|
|
processCategory(r, Category(cati), p.cmdLineRest, testsDir, runJoinableTests = false)
|
|
else:
|
|
addExitProc azure.finalize
|
|
quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
|
|
of "c", "cat", "category":
|
|
skips = loadSkipFrom(skipFrom)
|
|
var cat = Category(p.key)
|
|
processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true)
|
|
of "pcat":
|
|
skips = loadSkipFrom(skipFrom)
|
|
# 'pcat' is used for running a category in parallel. Currently the only
|
|
# difference is that we don't want to run joinable tests here as they
|
|
# are covered by the 'megatest' category.
|
|
isMainProcess = false
|
|
var cat = Category(p.key)
|
|
p.next
|
|
processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false)
|
|
of "p", "pat", "pattern":
|
|
skips = loadSkipFrom(skipFrom)
|
|
let pattern = p.key
|
|
p.next
|
|
processPattern(r, pattern, p.cmdLineRest, simulate)
|
|
of "r", "run":
|
|
let (cat, path) = splitTestFile(p.key)
|
|
processSingleTest(r, cat.Category, p.cmdLineRest, path, gTargets, targetsSet)
|
|
of "html":
|
|
generateHtml(resultsFile, optFailing)
|
|
else:
|
|
quit Usage
|
|
|
|
if optPrintResults:
|
|
if action == "html": openDefaultBrowser(resultsFile)
|
|
else: echo r, r.data
|
|
azure.finalize()
|
|
backend.close()
|
|
var failed = r.total - r.passed - r.skipped
|
|
if failed != 0:
|
|
echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ",
|
|
r.skipped, " failed: ", failed
|
|
quit(QuitFailure)
|
|
if isMainProcess:
|
|
echo "Used ", compilerPrefix, " to run the tests. Use --nim to override."
|
|
|
|
if paramCount() == 0:
|
|
quit Usage
|
|
main()
|