mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-04-19 21:10:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4070 from Feoramund/test-runner-mem-fail-as-error

Browse Source 
Let memory failures be errors in the test runner
This commit is contained in:
Jeroen van Rijn
2024-08-13 13:50:46 +02:00
committed by GitHub
parent 02b4bb8491 0af025d056
commit 069ad446cd
9 changed files with 143 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.github/workflows/ci.yml vendored
View File

@@ -32,10 +32,10 @@ jobs:
gmake -C vendor/miniaudio/src
./odin check examples/all -vet -strict-style -disallow-do -target:netbsd_amd64
./odin check examples/all -vet -strict-style -disallow-do -target:netbsd_arm64
./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
(cd tests/issues; ./run.sh)
build_freebsd:
name: FreeBSD Build, Check, and Test
@@ -61,10 +61,10 @@ jobs:
gmake -C vendor/cgltf/src
gmake -C vendor/miniaudio/src
./odin check examples/all -vet -strict-style -disallow-do -target:freebsd_amd64
./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
(cd tests/issues; ./run.sh)
ci:
strategy:
@@ -118,15 +118,15 @@ jobs:
- name: Odin check examples/all
run: ./odin check examples/all -strict-style
- name: Normal Core library tests
run: ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Optimized Core library tests
run: ./odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Vendor library tests
run: ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Internals tests
run: ./odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Core library benchmarks
run: ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: GitHub Issue tests
run: |
cd tests/issues
@@ -190,28 +190,28 @@ jobs:
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Optimized core library tests
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Core library benchmarks
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Vendor library tests
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
copy vendor\lua\5.4\windows\*.dll .
odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Odin internals tests
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Odin documentation tests
shell: cmd
run: |

8
core/testing/events.odin
View File

@@ -1,6 +1,14 @@
//+private
package testing
/*
(c) Copyright 2024 Feoramund <rune@swevencraft.org>.
Made available under Odin's BSD-3 license.
List of contributors:
Feoramund: Total rewrite.
*/
import "base:runtime"
import "core:sync/chan"
import "core:time"

9
core/testing/logging.odin
View File

@@ -1,6 +1,15 @@
//+private
package testing
/*
(c) Copyright 2024 Feoramund <rune@swevencraft.org>.
Made available under Odin's BSD-3 license.
List of contributors:
Ginger Bill: Initial implementation.
Feoramund: Total rewrite.
*/
import "base:runtime"
import "core:fmt"
import pkg_log "core:log"

8
core/testing/reporting.odin
View File

@@ -1,6 +1,14 @@
//+private
package testing
/*
(c) Copyright 2024 Feoramund <rune@swevencraft.org>.
Made available under Odin's BSD-3 license.
List of contributors:
Feoramund: Total rewrite.
*/
import "base:runtime"
import "core:encoding/ansi"
import "core:fmt"

107
core/testing/runner.odin
View File

@@ -1,6 +1,15 @@
//+private
package testing
/*
(c) Copyright 2024 Feoramund <rune@swevencraft.org>.
Made available under Odin's BSD-3 license.
List of contributors:
Ginger Bill: Initial implementation.
Feoramund: Total rewrite.
*/
import "base:intrinsics"
import "base:runtime"
import "core:bytes"
@@ -25,8 +34,8 @@ TEST_THREADS : int : #config(ODIN_TEST_THREADS, 0)
TRACKING_MEMORY : bool : #config(ODIN_TEST_TRACK_MEMORY, true)
// Always report how much memory is used, even when there are no leaks or bad frees.
ALWAYS_REPORT_MEMORY : bool : #config(ODIN_TEST_ALWAYS_REPORT_MEMORY, false)
// Log level for memory leaks and bad frees: debug, info, warning, error, fatal
LOG_LEVEL_MEMORY : string : #config(ODIN_TEST_LOG_LEVEL_MEMORY, "warning")
// Treat memory leaks and bad frees as errors.
FAIL_ON_BAD_MEMORY : bool : #config(ODIN_TEST_FAIL_ON_BAD_MEMORY, false)
// Specify how much memory each thread allocator starts with.
PER_THREAD_MEMORY : int : #config(ODIN_TEST_THREAD_MEMORY, mem.ROLLBACK_STACK_DEFAULT_BLOCK_SIZE)
// Select a specific set of tests to run by name.
@@ -65,21 +74,6 @@ get_log_level :: #force_inline proc() -> runtime.Logger_Level {
}
}
get_memory_log_level :: #force_inline proc() -> runtime.Logger_Level {
when ODIN_DEBUG {
// Always use .Debug in `-debug` mode.
return .Debug
} else {
when LOG_LEVEL_MEMORY == "debug" { return .Debug } else
when LOG_LEVEL_MEMORY == "info" { return .Info } else
when LOG_LEVEL_MEMORY == "warning" { return .Warning } else
when LOG_LEVEL_MEMORY == "error" { return .Error } else
when LOG_LEVEL_MEMORY == "fatal" { return .Fatal } else {
#panic("Unknown `ODIN_TEST_LOG_LEVEL_MEMORY`: \"" + LOG_LEVEL_MEMORY + "\", possible levels are: \"debug\", \"info\", \"warning\", \"error\", or \"fatal\".")
}
}
}
JSON :: struct {
total: int,
success: int,
@@ -103,10 +97,19 @@ end_t :: proc(t: ^T) {
t.cleanups = {}
}
Task_Data :: struct {
it: Internal_Test,
t: T,
allocator_index: int,
when TRACKING_MEMORY && FAIL_ON_BAD_MEMORY {
Task_Data :: struct {
it: Internal_Test,
t: T,
allocator_index: int,
tracking_allocator: ^mem.Tracking_Allocator,
}
} else {
Task_Data :: struct {
it: Internal_Test,
t: T,
allocator_index: int,
}
}
Task_Timeout :: struct {
@@ -150,6 +153,31 @@ run_test_task :: proc(task: thread.Task) {
end_t(&data.t)
when TRACKING_MEMORY && FAIL_ON_BAD_MEMORY {
// NOTE(Feoramund): The simplest way to handle treating memory failures
// as errors is to allow the test task runner to access the tracking
// allocator itself.
//
// This way, it's still able to send up a log message, which will be
// used in the end summary, and it can set the test state to `Failed`
// under the usual conditions.
//
// No outside intervention needed.
memory_leaks := len(data.tracking_allocator.allocation_map)
bad_frees := len(data.tracking_allocator.bad_free_array)
memory_is_in_bad_state := memory_leaks + bad_frees > 0
data.t.error_count += memory_leaks + bad_frees
if memory_is_in_bad_state {
pkg_log.errorf("Memory failure in `%s.%s` with %i leak%s and %i bad free%s.",
data.it.pkg, data.it.name,
memory_leaks, "" if memory_leaks == 1 else "s",
bad_frees, "" if bad_frees == 1 else "s")
}
}
new_state : Test_State = .Failed if failed(&data.t) else .Successful
chan.send(data.t.channel, Event_State_Change {
@@ -239,10 +267,6 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
total_success_count := 0
total_done_count := 0
total_test_count := len(internal_tests)
when TRACKING_MEMORY {
memory_leak_count := 0
bad_free_count := 0
}
when !FANCY_OUTPUT {
// This is strictly for updating the window title when the progress
@@ -439,6 +463,9 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
#no_bounds_check when TRACKING_MEMORY {
task_allocator := mem.tracking_allocator(&task_memory_trackers[task_index])
when FAIL_ON_BAD_MEMORY {
data.tracking_allocator = &task_memory_trackers[task_index]
}
} else {
task_allocator := mem.rollback_stack_allocator(&task_allocators[task_index])
}
@@ -485,8 +512,13 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
pkg_log.info("Memory tracking is enabled. Tests will log their memory usage if there's an issue.")
}
pkg_log.info("< Final Mem/ Total Mem> < Peak Mem> (#Free/Alloc) :: [package.test_name]")
} else when ALWAYS_REPORT_MEMORY {
pkg_log.warn("ODIN_TEST_ALWAYS_REPORT_MEMORY is true, but ODIN_TRACK_MEMORY is false.")
} else {
when ALWAYS_REPORT_MEMORY {
pkg_log.warn("ODIN_TEST_ALWAYS_REPORT_MEMORY is true, but ODIN_TEST_TRACK_MEMORY is false.")
}
when FAIL_ON_BAD_MEMORY {
pkg_log.warn("ODIN_TEST_FAIL_ON_BAD_MEMORY is true, but ODIN_TEST_TRACK_MEMORY is false.")
}
}
start_time := time.now()
@@ -519,9 +551,6 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
memory_is_in_bad_state := len(tracker.allocation_map) + len(tracker.bad_free_array) > 0
memory_leak_count += len(tracker.allocation_map)
bad_free_count += len(tracker.bad_free_array)
when ALWAYS_REPORT_MEMORY {
should_report := true
} else {
@@ -531,9 +560,11 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
if should_report {
write_memory_report(batch_writer, tracker, data.it.pkg, data.it.name)
memory_log_level := get_memory_log_level() if memory_is_in_bad_state else .Info
pkg_log.log(memory_log_level, bytes.buffer_to_string(&batch_buffer))
when FAIL_ON_BAD_MEMORY {
pkg_log.log(.Error if memory_is_in_bad_state else .Info, bytes.buffer_to_string(&batch_buffer))
} else {
pkg_log.log(.Warning if memory_is_in_bad_state else .Info, bytes.buffer_to_string(&batch_buffer))
}
bytes.buffer_reset(&batch_buffer)
}
@@ -917,11 +948,5 @@ To partly mitigate this, redirect STDERR to a file or use the -define:ODIN_TEST_
fmt.assertf(err == nil, "Error writing JSON report: %v", err)
}
fatal_memory_failures := false
when TRACKING_MEMORY {
if get_memory_log_level() >= .Error {
fatal_memory_failures = (memory_leak_count + bad_free_count) > 0
}
}
return total_success_count == total_test_count && !fatal_memory_failures
}
return total_success_count == total_test_count
}

8
core/testing/signal_handler.odin
View File

@@ -1,6 +1,14 @@
//+private
package testing
/*
(c) Copyright 2024 Feoramund <rune@swevencraft.org>.
Made available under Odin's BSD-3 license.
List of contributors:
Feoramund: Total rewrite.
*/
import "base:runtime"
import pkg_log "core:log"

8
core/testing/signal_handler_libc.odin
View File

@@ -2,6 +2,14 @@
//+build windows, linux, darwin, freebsd, openbsd, netbsd, haiku
package testing
/*
(c) Copyright 2024 Feoramund <rune@swevencraft.org>.
Made available under Odin's BSD-3 license.
List of contributors:
Feoramund: Total rewrite.
*/
import "base:intrinsics"
import "core:c/libc"
import "core:encoding/ansi"

8
core/testing/signal_handler_other.odin
View File

@@ -2,6 +2,14 @@
//+build !windows !linux !darwin !freebsd !openbsd !netbsd !haiku
package testing
/*
(c) Copyright 2024 Feoramund <rune@swevencraft.org>.
Made available under Odin's BSD-3 license.
List of contributors:
Feoramund: Total rewrite.
*/
_setup_signal_handler :: proc() {
// Do nothing.
}

11
core/testing/testing.odin
View File

@@ -1,5 +1,14 @@
package testing
/*
(c) Copyright 2024 Feoramund <rune@swevencraft.org>.
Made available under Odin's BSD-3 license.
List of contributors:
Ginger Bill: Initial implementation.
Feoramund: Total rewrite.
*/
import "base:intrinsics"
import "base:runtime"
import pkg_log "core:log"
@@ -162,4 +171,4 @@ set_fail_timeout :: proc(t: ^T, duration: time.Duration, loc := #caller_location
at_time = time.time_add(time.now(), duration),
location = loc,
})
}
}