mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-04-18 12:30:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optionally treat memory failures as errors in the test runner

Browse Source 
Enable with `-define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true`.
This commit is contained in:
Feoramund
2024-08-11 22:59:20 -04:00
parent 1761802330
commit 675add4d90
2 changed files with 75 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.github/workflows/ci.yml vendored
View File

@@ -30,10 +30,10 @@ jobs:
gmake -C vendor/miniaudio/src
./odin check examples/all -vet -strict-style -disallow-do -target:netbsd_amd64
./odin check examples/all -vet -strict-style -disallow-do -target:netbsd_arm64
./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
(cd tests/issues; ./run.sh)
build_freebsd:
name: FreeBSD Build, Check, and Test
@@ -59,10 +59,10 @@ jobs:
gmake -C vendor/cgltf/src
gmake -C vendor/miniaudio/src
./odin check examples/all -vet -strict-style -disallow-do -target:freebsd_amd64
./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
(cd tests/issues; ./run.sh)
ci:
strategy:
@@ -116,15 +116,15 @@ jobs:
- name: Odin check examples/all
run: ./odin check examples/all -strict-style
- name: Normal Core library tests
run: ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Optimized Core library tests
run: ./odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Vendor library tests
run: ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Internals tests
run: ./odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Core library benchmarks
run: ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
run: ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: GitHub Issue tests
run: |
cd tests/issues
@@ -188,28 +188,28 @@ jobs:
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Optimized core library tests
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Core library benchmarks
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Vendor library tests
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
copy vendor\lua\5.4\windows\*.dll .
odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Odin internals tests
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal
odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true
- name: Odin documentation tests
shell: cmd
run: |

98
core/testing/runner.odin
View File

@@ -25,8 +25,8 @@ TEST_THREADS : int : #config(ODIN_TEST_THREADS, 0)
TRACKING_MEMORY : bool : #config(ODIN_TEST_TRACK_MEMORY, true)
// Always report how much memory is used, even when there are no leaks or bad frees.
ALWAYS_REPORT_MEMORY : bool : #config(ODIN_TEST_ALWAYS_REPORT_MEMORY, false)
// Log level for memory leaks and bad frees: debug, info, warning, error, fatal
LOG_LEVEL_MEMORY : string : #config(ODIN_TEST_LOG_LEVEL_MEMORY, "warning")
// Treat memory leaks and bad frees as errors.
FAIL_ON_BAD_MEMORY : bool : #config(ODIN_TEST_FAIL_ON_BAD_MEMORY, false)
// Specify how much memory each thread allocator starts with.
PER_THREAD_MEMORY : int : #config(ODIN_TEST_THREAD_MEMORY, mem.ROLLBACK_STACK_DEFAULT_BLOCK_SIZE)
// Select a specific set of tests to run by name.
@@ -65,21 +65,6 @@ get_log_level :: #force_inline proc() -> runtime.Logger_Level {
}
}
get_memory_log_level :: #force_inline proc() -> runtime.Logger_Level {
when ODIN_DEBUG {
// Always use .Debug in `-debug` mode.
return .Debug
} else {
when LOG_LEVEL_MEMORY == "debug" { return .Debug } else
when LOG_LEVEL_MEMORY == "info" { return .Info } else
when LOG_LEVEL_MEMORY == "warning" { return .Warning } else
when LOG_LEVEL_MEMORY == "error" { return .Error } else
when LOG_LEVEL_MEMORY == "fatal" { return .Fatal } else {
#panic("Unknown `ODIN_TEST_LOG_LEVEL_MEMORY`: \"" + LOG_LEVEL_MEMORY + "\", possible levels are: \"debug\", \"info\", \"warning\", \"error\", or \"fatal\".")
}
}
}
JSON :: struct {
total: int,
success: int,
@@ -103,10 +88,19 @@ end_t :: proc(t: ^T) {
t.cleanups = {}
}
Task_Data :: struct {
it: Internal_Test,
t: T,
allocator_index: int,
when TRACKING_MEMORY && FAIL_ON_BAD_MEMORY {
Task_Data :: struct {
it: Internal_Test,
t: T,
allocator_index: int,
tracking_allocator: ^mem.Tracking_Allocator,
}
} else {
Task_Data :: struct {
it: Internal_Test,
t: T,
allocator_index: int,
}
}
Task_Timeout :: struct {
@@ -150,6 +144,31 @@ run_test_task :: proc(task: thread.Task) {
end_t(&data.t)
when TRACKING_MEMORY && FAIL_ON_BAD_MEMORY {
// NOTE(Feoramund): The simplest way to handle treating memory failures
// as errors is to allow the test task runner to access the tracking
// allocator itself.
//
// This way, it's still able to send up a log message, which will be
// used in the end summary, and it can set the test state to `Failed`
// under the usual conditions.
//
// No outside intervention needed.
memory_leaks := len(data.tracking_allocator.allocation_map)
bad_frees := len(data.tracking_allocator.bad_free_array)
memory_is_in_bad_state := memory_leaks + bad_frees > 0
data.t.error_count += memory_leaks + bad_frees
if memory_is_in_bad_state {
pkg_log.errorf("Memory failure in `%s.%s` with %i leak%s and %i bad free%s.",
data.it.pkg, data.it.name,
memory_leaks, "" if memory_leaks == 1 else "s",
bad_frees, "" if bad_frees == 1 else "s")
}
}
new_state : Test_State = .Failed if failed(&data.t) else .Successful
chan.send(data.t.channel, Event_State_Change {
@@ -239,10 +258,6 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
total_success_count := 0
total_done_count := 0
total_test_count := len(internal_tests)
when TRACKING_MEMORY {
memory_leak_count := 0
bad_free_count := 0
}
when !FANCY_OUTPUT {
// This is strictly for updating the window title when the progress
@@ -439,6 +454,9 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
#no_bounds_check when TRACKING_MEMORY {
task_allocator := mem.tracking_allocator(&task_memory_trackers[task_index])
when FAIL_ON_BAD_MEMORY {
data.tracking_allocator = &task_memory_trackers[task_index]
}
} else {
task_allocator := mem.rollback_stack_allocator(&task_allocators[task_index])
}
@@ -485,8 +503,13 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
pkg_log.info("Memory tracking is enabled. Tests will log their memory usage if there's an issue.")
}
pkg_log.info("< Final Mem/ Total Mem> < Peak Mem> (#Free/Alloc) :: [package.test_name]")
} else when ALWAYS_REPORT_MEMORY {
pkg_log.warn("ODIN_TEST_ALWAYS_REPORT_MEMORY is true, but ODIN_TRACK_MEMORY is false.")
} else {
when ALWAYS_REPORT_MEMORY {
pkg_log.warn("ODIN_TEST_ALWAYS_REPORT_MEMORY is true, but ODIN_TRACK_MEMORY is false.")
}
when FAIL_ON_BAD_MEMORY {
pkg_log.warn("ODIN_TEST_FAIL_ON_BAD_MEMORY is true, but ODIN_TRACK_MEMORY is false.")
}
}
start_time := time.now()
@@ -519,9 +542,6 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
memory_is_in_bad_state := len(tracker.allocation_map) + len(tracker.bad_free_array) > 0
memory_leak_count += len(tracker.allocation_map)
bad_free_count += len(tracker.bad_free_array)
when ALWAYS_REPORT_MEMORY {
should_report := true
} else {
@@ -531,9 +551,11 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
if should_report {
write_memory_report(batch_writer, tracker, data.it.pkg, data.it.name)
memory_log_level := get_memory_log_level() if memory_is_in_bad_state else .Info
pkg_log.log(memory_log_level, bytes.buffer_to_string(&batch_buffer))
when FAIL_ON_BAD_MEMORY {
pkg_log.log(.Error if memory_is_in_bad_state else .Info, bytes.buffer_to_string(&batch_buffer))
} else {
pkg_log.log(.Warning if memory_is_in_bad_state else .Info, bytes.buffer_to_string(&batch_buffer))
}
bytes.buffer_reset(&batch_buffer)
}
@@ -917,11 +939,5 @@ To partly mitigate this, redirect STDERR to a file or use the -define:ODIN_TEST_
fmt.assertf(err == nil, "Error writing JSON report: %v", err)
}
fatal_memory_failures := false
when TRACKING_MEMORY {
if get_memory_log_level() >= .Error {
fatal_memory_failures = (memory_leak_count + bad_free_count) > 0
}
}
return total_success_count == total_test_count && !fatal_memory_failures
}
return total_success_count == total_test_count
}