diff --git a/core/testing/runner.odin b/core/testing/runner.odin index 56d561d3d..cb1da9445 100644 --- a/core/testing/runner.odin +++ b/core/testing/runner.odin @@ -741,7 +741,8 @@ runner :: proc(internal_tests: []Internal_Test) -> bool { if test_index, reason, ok := should_stop_test(); ok { - #no_bounds_check report.all_test_states[test_index] = .Failed + passed := reason == .Successful_Stop + #no_bounds_check report.all_test_states[test_index] = .Successful if passed else .Failed #no_bounds_check it := internal_tests[test_index] #no_bounds_check pkg := report.packages_by_name[it.pkg] pkg.frame_ready = false @@ -762,7 +763,7 @@ runner :: proc(internal_tests: []Internal_Test) -> bool { fmt.assertf(task_data != nil, "A signal (%v) was raised to stop test #%i %s.%s, but its task data is missing.", reason, test_index, it.pkg, it.name) - if !task_data.t._fail_now_called { + if !passed && !task_data.t._fail_now_called { if test_index not_in failed_test_reason_map { // We only write a new error message here if there wasn't one // already, because the message we can provide based only on @@ -780,7 +781,11 @@ runner :: proc(internal_tests: []Internal_Test) -> bool { end_t(&task_data.t) - total_failure_count += 1 + if passed { + total_success_count += 1 + } else { + total_failure_count += 1 + } total_done_count += 1 } diff --git a/core/testing/signal_handler.odin b/core/testing/signal_handler.odin index 2f1f7c89a..73ed362f0 100644 --- a/core/testing/signal_handler.odin +++ b/core/testing/signal_handler.odin @@ -12,8 +12,26 @@ package testing import "base:runtime" import "core:log" +@(private, thread_local) +local_test_expected_failures: struct { + signal: i32, + + message_count: int, + messages: [MAX_EXPECTED_ASSERTIONS_PER_TEST]string, + + location_count: int, + locations: [MAX_EXPECTED_ASSERTIONS_PER_TEST]runtime.Source_Code_Location, +} + +@(private, thread_local) +local_test_assertion_raised: struct { + message: string, + location: runtime.Source_Code_Location, +} + Stop_Reason :: enum { Unknown, + Successful_Stop, Illegal_Instruction, Arithmetic_Error, Segmentation_Fault, @@ -21,7 +39,12 @@ Stop_Reason :: enum { } test_assertion_failure_proc :: proc(prefix, message: string, loc: runtime.Source_Code_Location) -> ! { - log.fatalf("%s: %s", prefix, message, location = loc) + if local_test_expected_failures.message_count + local_test_expected_failures.location_count > 0 { + local_test_assertion_raised = { message, loc } + log.debugf("%s\n\tmessage: %q\n\tlocation: %w", prefix, message, loc) + } else { + log.fatalf("%s: %s", prefix, message, location = loc) + } runtime.trap() } diff --git a/core/testing/signal_handler_libc.odin b/core/testing/signal_handler_libc.odin index f9527e22f..4fc9552ae 100644 --- a/core/testing/signal_handler_libc.odin +++ b/core/testing/signal_handler_libc.odin @@ -20,7 +20,8 @@ import "core:terminal/ansi" @(private="file") stop_test_gate: sync.Mutex @(private="file") stop_test_index: libc.sig_atomic_t -@(private="file") stop_test_reason: libc.sig_atomic_t +@(private="file") stop_test_signal: libc.sig_atomic_t +@(private="file") stop_test_passed: libc.sig_atomic_t @(private="file") stop_test_alert: libc.sig_atomic_t @(private="file", thread_local) @@ -99,7 +100,30 @@ This is a dire bug and should be reported to the Odin developers. if sync.mutex_guard(&stop_test_gate) { intrinsics.atomic_store(&stop_test_index, local_test_index) - intrinsics.atomic_store(&stop_test_reason, cast(libc.sig_atomic_t)sig) + intrinsics.atomic_store(&stop_test_signal, cast(libc.sig_atomic_t)sig) + passed: bool + check_passing: { + if location := local_test_assertion_raised.location; location != {} { + for i in 0.. (test_index: int, reason: Stop_Reason, ok: bool) intrinsics.atomic_store(&stop_test_alert, 0) test_index = cast(int)intrinsics.atomic_load(&stop_test_index) - switch intrinsics.atomic_load(&stop_test_reason) { - case libc.SIGFPE: reason = .Arithmetic_Error - case libc.SIGILL: reason = .Illegal_Instruction - case libc.SIGSEGV: reason = .Segmentation_Fault - case SIGTRAP: reason = .Unhandled_Trap + if cast(bool)intrinsics.atomic_load(&stop_test_passed) { + reason = .Successful_Stop + } else { + switch intrinsics.atomic_load(&stop_test_signal) { + case libc.SIGFPE: reason = .Arithmetic_Error + case libc.SIGILL: reason = .Illegal_Instruction + case libc.SIGSEGV: reason = .Segmentation_Fault + case SIGTRAP: reason = .Unhandled_Trap + } } ok = true } diff --git a/core/testing/testing.odin b/core/testing/testing.odin index 09bf6dc0e..1357a4683 100644 --- a/core/testing/testing.odin +++ b/core/testing/testing.odin @@ -21,6 +21,8 @@ import "core:mem" _ :: reflect // alias reflect to nothing to force visibility for -vet _ :: mem // in case TRACKING_MEMORY is not enabled +MAX_EXPECTED_ASSERTIONS_PER_TEST :: 5 + // IMPORTANT NOTE: Compiler requires this layout Test_Signature :: proc(^T) @@ -155,3 +157,74 @@ set_fail_timeout :: proc(t: ^T, duration: time.Duration, loc := #caller_location location = loc, }) } + +/* +Let the test runner know that it should expect an assertion failure from a +specific location in the source code for this test. + +In the event that an assertion fails, a debug message will be logged with its +exact message and location in a copyable format to make it convenient to write +tests which use this API. + +This procedure may be called up to 5 times with different locations. + +This is a limitation for the sake of simplicity in the implementation, and you +should consider breaking up your tests into smaller procedures if you need to +check for asserts in more than 2 places. +*/ +expect_assert_from :: proc(t: ^T, expected_place: runtime.Source_Code_Location, caller_loc := #caller_location) { + count := local_test_expected_failures.location_count + if count == MAX_EXPECTED_ASSERTIONS_PER_TEST { + panic("This test cannot handle that many expected assertions based on matching the location.", caller_loc) + } + local_test_expected_failures.locations[count] = expected_place + local_test_expected_failures.location_count += 1 +} + +/* +Let the test runner know that it should expect an assertion failure with a +specific message for this test. + +In the event that an assertion fails, a debug message will be logged with its +exact message and location in a copyable format to make it convenient to write +tests which use this API. + +This procedure may be called up to 5 times with different messages. + +This is a limitation for the sake of simplicity in the implementation, and you +should consider breaking up your tests into smaller procedures if you need to +check for more than a couple different assertion messages. +*/ +expect_assert_message :: proc(t: ^T, expected_message: string, caller_loc := #caller_location) { + count := local_test_expected_failures.message_count + if count == MAX_EXPECTED_ASSERTIONS_PER_TEST { + panic("This test cannot handle that many expected assertions based on matching the message.", caller_loc) + } + local_test_expected_failures.messages[count] = expected_message + local_test_expected_failures.message_count += 1 +} + +expect_assert :: proc { + expect_assert_from, + expect_assert_message, +} + +/* +Let the test runner know that it should expect a signal to be raised within +this test. + +This API is for advanced users, as arbitrary signals will not be caught; only +the ones already handled by the test runner, such as + +- SIGINT, (interrupt) +- SIGTERM, (polite termination) +- SIGILL, (illegal instruction) +- SIGFPE, (arithmetic error) +- SIGSEGV, and (segmentation fault) +- SIGTRAP (only on POSIX systems). (trap / debug trap) + +Note that only one signal can be expected per test. +*/ +expect_signal :: proc(t: ^T, #any_int sig: i32) { + local_test_expected_failures.signal = sig +}