Allow testing for intentional leaks in test runner

Adds `expect_leak_or_bad_free :: proc(t: ^T, client_test: proc(t: ^T), verifier: Memory_Verifier_Proc)`.

It sets up its own `Tracking_Allocator`, runs the `client_test`, and then calls the `verifier` procedure.
The verifier can then inspect the contents of the tracking allocator and call `testing.expect*` as sensible for the test in question.

Any allocations are then cleared so that the test runner doesn't itself complain about leaks.

Additionally, `ODIN_TEST_LOG_LEVEL_MEMORY` has been added as a define to set the severity of the test runner's memory tracker. You can use `-define:ODIN_TEST_LOG_LEVEL_MEMORY=error` to make tests fail rather than warn if leaks or bad frees have been found.

core/testing/runner.odin

@@ -25,6 +25,8 @@ TEST_THREADS          : int    : #config(ODIN_TEST_THREADS, 0)
 TRACKING_MEMORY       : bool   : #config(ODIN_TEST_TRACK_MEMORY, true)
 // Always report how much memory is used, even when there are no leaks or bad frees.
 ALWAYS_REPORT_MEMORY  : bool   : #config(ODIN_TEST_ALWAYS_REPORT_MEMORY, false)
+// Log level for memory leaks and bad frees: debug, info, warning, error, fatal
+LOG_LEVEL_MEMORY      : string : #config(ODIN_TEST_LOG_LEVEL_MEMORY, "warning")
 // Specify how much memory each thread allocator starts with.
 PER_THREAD_MEMORY     : int    : #config(ODIN_TEST_THREAD_MEMORY, mem.ROLLBACK_STACK_DEFAULT_BLOCK_SIZE)
 // Select a specific set of tests to run by name.
@@ -63,6 +65,21 @@ get_log_level :: #force_inline proc() -> runtime.Logger_Level {
 	}
 }
 
+get_memory_log_level :: #force_inline proc() -> runtime.Logger_Level {
+	when ODIN_DEBUG {
+		// Always use .Debug in `-debug` mode.
+		return .Debug
+	} else {
+		when LOG_LEVEL_MEMORY == "debug"   { return .Debug   } else
+		when LOG_LEVEL_MEMORY == "info"    { return .Info    } else
+		when LOG_LEVEL_MEMORY == "warning" { return .Warning } else
+		when LOG_LEVEL_MEMORY == "error"   { return .Error   } else
+		when LOG_LEVEL_MEMORY == "fatal"   { return .Fatal   } else {
+			#panic("Unknown `ODIN_TEST_LOG_LEVEL_MEMORY`: \"" + LOG_LEVEL_MEMORY + "\", possible levels are: \"debug\", \"info\", \"warning\", \"error\", or \"fatal\".")
+		}
+	}
+}
+
 JSON :: struct {
 	total:    int,
 	success:  int,
@@ -222,6 +239,10 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
 	total_success_count := 0
 	total_done_count    := 0
 	total_test_count    := len(internal_tests)
+	when TRACKING_MEMORY {
+		memory_leak_count   := 0
+		bad_free_count      := 0
+	}
 
 	when !FANCY_OUTPUT {
 		// This is strictly for updating the window title when the progress
@@ -498,6 +519,9 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
 
 				memory_is_in_bad_state := len(tracker.allocation_map) + len(tracker.bad_free_array) > 0
 
+				memory_leak_count += len(tracker.allocation_map)
+				bad_free_count    += len(tracker.bad_free_array)
+
 				when ALWAYS_REPORT_MEMORY {
 					should_report := true
 				} else {
@@ -507,7 +531,9 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
 				if should_report {
 					write_memory_report(batch_writer, tracker, data.it.pkg, data.it.name)
 
-					pkg_log.log(.Warning if memory_is_in_bad_state else .Info, bytes.buffer_to_string(&batch_buffer))
+					memory_log_level := get_memory_log_level() if memory_is_in_bad_state else .Info
+
+					pkg_log.log(memory_log_level, bytes.buffer_to_string(&batch_buffer))
 					bytes.buffer_reset(&batch_buffer)
 				}
 
@@ -891,5 +917,11 @@ To partly mitigate this, redirect STDERR to a file or use the -define:ODIN_TEST_
 		fmt.assertf(err == nil, "Error writing JSON report: %v", err)
 	}
 
-	return total_success_count == total_test_count
-}
+	fatal_memory_failures := false
+	when TRACKING_MEMORY {
+		if get_memory_log_level() >= .Error {
+			fatal_memory_failures = (memory_leak_count + bad_free_count) > 0
+		}
+	}
+	return total_success_count == total_test_count && !fatal_memory_failures
+}

core/testing/testing.odin

@@ -4,8 +4,10 @@ import "base:intrinsics"
 import "base:runtime"
 import pkg_log "core:log"
 import "core:reflect"
+import "core:sync"
 import "core:sync/chan"
 import "core:time"
+import "core:mem"
 
 _ :: reflect // alias reflect to nothing to force visibility for -vet
 
@@ -136,10 +138,28 @@ expect_value :: proc(t: ^T, value, expected: $T, loc := #caller_location) -> boo
 	return ok
 }
 
+Memory_Verifier_Proc :: #type proc(t: ^T, ta: ^mem.Tracking_Allocator)
+
+expect_leaks :: proc(t: ^T, client_test: proc(t: ^T), verifier: Memory_Verifier_Proc) {
+	{
+		ta: mem.Tracking_Allocator
+		mem.tracking_allocator_init(&ta, context.allocator)
+		defer mem.tracking_allocator_destroy(&ta)
+		context.allocator = mem.tracking_allocator(&ta)
+
+		client_test(t)
+		sync.mutex_lock(&ta.mutex)
+		// The verifier can inspect this local tracking allocator.
+		// And then call `testing.expect_*` as makes sense for the client test.
+		verifier(t, &ta)
+		sync.mutex_unlock(&ta.mutex)
+	}
+	free_all(context.allocator)
+}
 
 set_fail_timeout :: proc(t: ^T, duration: time.Duration, loc := #caller_location) {
 	chan.send(t.channel, Event_Set_Fail_Timeout {
 		at_time = time.time_add(time.now(), duration),
 		location = loc,
 	})
-}
+}

tests/core/mem/test_mem_dynamic_pool.odin

@@ -5,76 +5,91 @@ import "core:mem"
 
 
 expect_pool_allocation :: proc(t: ^testing.T, expected_used_bytes, num_bytes, alignment: int) {
-    pool: mem.Dynamic_Pool
-    mem.dynamic_pool_init(pool = &pool, alignment = alignment)
-    pool_allocator := mem.dynamic_pool_allocator(&pool)
+	pool: mem.Dynamic_Pool
+	mem.dynamic_pool_init(pool = &pool, alignment = alignment)
+	pool_allocator := mem.dynamic_pool_allocator(&pool)
 
-    element, err := mem.alloc(num_bytes, alignment, pool_allocator)
-    testing.expect(t, err == .None)
-    testing.expect(t, element != nil)
+	element, err := mem.alloc(num_bytes, alignment, pool_allocator)
+	testing.expect(t, err == .None)
+	testing.expect(t, element != nil)
 
-    expected_bytes_left := pool.block_size - expected_used_bytes
-    testing.expectf(t, pool.bytes_left == expected_bytes_left,
-        `
-        Allocated data with size %v bytes, expected %v bytes left, got %v bytes left, off by %v bytes.
-        Pool:
-        block_size = %v
-        out_band_size = %v
-        alignment = %v
-        unused_blocks = %v
-        used_blocks = %v
-        out_band_allocations = %v
-        current_block = %v
-        current_pos = %v
-        bytes_left = %v
-        `,
-        num_bytes, expected_bytes_left, pool.bytes_left, expected_bytes_left - pool.bytes_left,
-        pool.block_size,
-        pool.out_band_size,
-        pool.alignment,
-        pool.unused_blocks,
-        pool.used_blocks,
-        pool.out_band_allocations,
-        pool.current_block,
-        pool.current_pos,
-        pool.bytes_left,
-    )
+	expected_bytes_left := pool.block_size - expected_used_bytes
+	testing.expectf(t, pool.bytes_left == expected_bytes_left,
+		`
+		Allocated data with size %v bytes, expected %v bytes left, got %v bytes left, off by %v bytes.
+		Pool:
+		block_size = %v
+		out_band_size = %v
+		alignment = %v
+		unused_blocks = %v
+		used_blocks = %v
+		out_band_allocations = %v
+		current_block = %v
+		current_pos = %v
+		bytes_left = %v
+		`,
+		num_bytes, expected_bytes_left, pool.bytes_left, expected_bytes_left - pool.bytes_left,
+		pool.block_size,
+		pool.out_band_size,
+		pool.alignment,
+		pool.unused_blocks,
+		pool.used_blocks,
+		pool.out_band_allocations,
+		pool.current_block,
+		pool.current_pos,
+		pool.bytes_left,
+	)
 
-    mem.dynamic_pool_destroy(&pool)
-    testing.expect(t, pool.used_blocks == nil)
+	mem.dynamic_pool_destroy(&pool)
+	testing.expect(t, pool.used_blocks == nil)
 }
 
 expect_pool_allocation_out_of_band :: proc(t: ^testing.T, num_bytes, out_band_size: int) {
-    testing.expect(t, num_bytes >= out_band_size, "Sanity check failed, your test call is flawed! Make sure that num_bytes >= out_band_size!")
+	testing.expect(t, num_bytes >= out_band_size, "Sanity check failed, your test call is flawed! Make sure that num_bytes >= out_band_size!")
 
-    pool: mem.Dynamic_Pool
-    mem.dynamic_pool_init(pool = &pool, out_band_size = out_band_size)
-    pool_allocator := mem.dynamic_pool_allocator(&pool)
+	pool: mem.Dynamic_Pool
+	mem.dynamic_pool_init(pool = &pool, out_band_size = out_band_size)
+	pool_allocator := mem.dynamic_pool_allocator(&pool)
 
-    element, err := mem.alloc(num_bytes, allocator = pool_allocator)
-    testing.expect(t, err == .None)
-    testing.expect(t, element != nil)
-    testing.expectf(t, pool.out_band_allocations != nil,
-        "Allocated data with size %v bytes, which is >= out_of_band_size and it should be in pool.out_band_allocations, but isn't!",
-    )
+	element, err := mem.alloc(num_bytes, allocator = pool_allocator)
+	testing.expect(t, err == .None)
+	testing.expect(t, element != nil)
+	testing.expectf(t, pool.out_band_allocations != nil,
+		"Allocated data with size %v bytes, which is >= out_of_band_size and it should be in pool.out_band_allocations, but isn't!",
+	)
 
-    mem.dynamic_pool_destroy(&pool)
-    testing.expect(t, pool.out_band_allocations == nil)
+	mem.dynamic_pool_destroy(&pool)
+	testing.expect(t, pool.out_band_allocations == nil)
 }
 
 @(test)
 test_dynamic_pool_alloc_aligned :: proc(t: ^testing.T) {
-    expect_pool_allocation(t, expected_used_bytes = 16, num_bytes = 16, alignment=8)
+	expect_pool_allocation(t, expected_used_bytes = 16, num_bytes = 16, alignment=8)
 }
 
 @(test)
 test_dynamic_pool_alloc_unaligned :: proc(t: ^testing.T) {
-    expect_pool_allocation(t, expected_used_bytes =   8,   num_bytes=1, alignment=8)
-    expect_pool_allocation(t, expected_used_bytes =   16,  num_bytes=9, alignment=8)
+	expect_pool_allocation(t, expected_used_bytes =   8,   num_bytes=1, alignment=8)
+	expect_pool_allocation(t, expected_used_bytes =   16,  num_bytes=9, alignment=8)
 }
 
 @(test)
 test_dynamic_pool_alloc_out_of_band :: proc(t: ^testing.T) {
-    expect_pool_allocation_out_of_band(t, num_bytes = 128, out_band_size = 128)
-    expect_pool_allocation_out_of_band(t, num_bytes = 129, out_band_size = 128)
+	expect_pool_allocation_out_of_band(t, num_bytes = 128, out_band_size = 128)
+	expect_pool_allocation_out_of_band(t, num_bytes = 129, out_band_size = 128)
+}
+
+@(test)
+test_intentional_leaks :: proc(t: ^testing.T) {
+	testing.expect_leaks(t, intentionally_leaky_test, leak_verifier)
+}
+
+// Not tagged with @(test) because it's run through `test_intentional_leaks`
+intentionally_leaky_test :: proc(t: ^testing.T) {
+	a: [dynamic]int
+	append(&a, 42)
+}
+
+leak_verifier :: proc(t: ^testing.T, ta: ^mem.Tracking_Allocator) {
+	testing.expect_value(t, len(ta.allocation_map), 1)
 }