From c52c29856c22aaf4e0135bd4e800be8b937640d8 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Thu, 6 Feb 2025 19:20:30 -0500 Subject: [PATCH] Unify heap allocators - Simplify the malloc-based allocator - Correct the type signatures on malloc/calloc/realloc - Make `heap_alloc/heap_free/heap_resize` API consistent across platforms --- base/runtime/default_allocators_general.odin | 3 - base/runtime/heap_allocator.odin | 4 +- base/runtime/heap_allocator_fallback.odin | 145 +++++++++--------- .../heap_allocator_implementation.odin | 22 +-- base/runtime/wasm_allocator.odin | 20 +++ 5 files changed, 104 insertions(+), 90 deletions(-) diff --git a/base/runtime/default_allocators_general.odin b/base/runtime/default_allocators_general.odin index 64af6c904..cbaf4d22a 100644 --- a/base/runtime/default_allocators_general.odin +++ b/base/runtime/default_allocators_general.odin @@ -6,9 +6,6 @@ when ODIN_DEFAULT_TO_NIL_ALLOCATOR { } else when ODIN_DEFAULT_TO_PANIC_ALLOCATOR { default_allocator_proc :: panic_allocator_proc default_allocator :: panic_allocator -} else when ODIN_OS != .Orca && (ODIN_ARCH == .wasm32 || ODIN_ARCH == .wasm64p32) { - default_allocator :: default_wasm_allocator - default_allocator_proc :: wasm_allocator_proc } else { default_allocator :: heap_allocator default_allocator_proc :: heap_allocator_proc diff --git a/base/runtime/heap_allocator.odin b/base/runtime/heap_allocator.odin index 0fb160347..7fc1c45c6 100644 --- a/base/runtime/heap_allocator.odin +++ b/base/runtime/heap_allocator.odin @@ -35,7 +35,7 @@ heap_allocator_proc :: proc( } return transmute([]byte)Raw_Slice{ data = ptr, len = size }, nil case .Alloc_Non_Zeroed: - ptr := heap_alloc(max(size, alignment), zero = false) + ptr := heap_alloc(max(size, alignment), zero_memory = false) if ptr == nil { return nil, .Out_Of_Memory } @@ -47,7 +47,7 @@ heap_allocator_proc :: proc( } return transmute([]byte)Raw_Slice{ data = ptr, len = size }, nil case .Resize_Non_Zeroed: - ptr := heap_resize(old_memory, old_size, max(size, alignment), zero = false) + ptr := heap_resize(old_memory, old_size, max(size, alignment), zero_memory = false) if ptr == nil { return nil, .Out_Of_Memory } diff --git a/base/runtime/heap_allocator_fallback.odin b/base/runtime/heap_allocator_fallback.odin index 129cd9d86..25e906d7e 100644 --- a/base/runtime/heap_allocator_fallback.odin +++ b/base/runtime/heap_allocator_fallback.odin @@ -9,25 +9,39 @@ package runtime import "base:intrinsics" foreign { - @(link_name="malloc") _libc_malloc :: proc "c" (size: int) -> rawptr --- - @(link_name="calloc") _libc_calloc :: proc "c" (num, size: int) -> rawptr --- + @(link_name="malloc") _libc_malloc :: proc "c" (size: uint) -> rawptr --- + @(link_name="calloc") _libc_calloc :: proc "c" (num, size: uint) -> rawptr --- @(link_name="free") _libc_free :: proc "c" (ptr: rawptr) --- - @(link_name="realloc") _libc_realloc :: proc "c" (ptr: rawptr, size: int) -> rawptr --- + @(link_name="realloc") _libc_realloc :: proc "c" (ptr: rawptr, size: uint) -> rawptr --- } +@(require_results) heap_alloc :: proc "contextless" (size: int, zero_memory := true) -> rawptr { if size <= 0 { return nil } if zero_memory { - return _libc_calloc(1, size) + return _libc_calloc(1, uint(size)) } else { - return _libc_malloc(size) + return _libc_malloc(uint(size)) } } -heap_resize :: proc "contextless" (ptr: rawptr, new_size: int) -> rawptr { - return _libc_realloc(ptr, new_size) +@(require_results) +heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int, zero_memory: bool = true) -> (new_ptr: rawptr) { + new_ptr = _libc_realloc(old_ptr, uint(new_size)) + + // Section 7.22.3.5.2 of the C17 standard: "The contents of the new object + // shall be the same as that of the old object prior to deallocation, up to + // the lesser of the new and old sizes. Any bytes in the new object beyond + // the size of the old object have indeterminate values." + // + // Therefore, we zero the memory ourselves. + if zero_memory && new_size > old_size { + intrinsics.mem_zero(rawptr(uintptr(new_ptr) + uintptr(old_size)), new_size - old_size) + } + + return } heap_free :: proc "contextless" (ptr: rawptr) { @@ -44,84 +58,67 @@ heap_allocator :: proc() -> Allocator { heap_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, size, alignment: int, old_memory: rawptr, old_size: int, loc := #caller_location) -> ([]byte, Allocator_Error) { - // - // NOTE(tetra, 2020-01-14): The heap doesn't respect alignment. - // Instead, we overallocate by `alignment + size_of(rawptr) - 1`, and insert - // padding. We also store the original pointer returned by heap_alloc right before - // the pointer we return to the user. - // - aligned_alloc :: proc(size, alignment: int, old_ptr: rawptr, old_size: int, zero_memory := true) -> ([]byte, Allocator_Error) { - // NOTE(flysand): We need to reserve enough space for alignment, which - // includes the user data itself, the space to store the pointer to - // allocation start, as well as the padding required to align both - // the user data and the pointer. - a := max(alignment, align_of(rawptr)) - space := a-1 + size_of(rawptr) + size - allocated_mem: rawptr - - force_copy := old_ptr != nil && alignment > align_of(rawptr) - - if old_ptr != nil && !force_copy { - original_old_ptr := ([^]rawptr)(old_ptr)[-1] - allocated_mem = heap_resize(original_old_ptr, space) - } else { - allocated_mem = heap_alloc(space, zero_memory) - } - aligned_mem := rawptr(([^]u8)(allocated_mem)[size_of(rawptr):]) - - ptr := uintptr(aligned_mem) - aligned_ptr := (ptr + uintptr(a)-1) & ~(uintptr(a)-1) - if allocated_mem == nil { - aligned_free(old_ptr) - aligned_free(allocated_mem) - return nil, .Out_Of_Memory - } - - aligned_mem = rawptr(aligned_ptr) - ([^]rawptr)(aligned_mem)[-1] = allocated_mem - - if force_copy { - mem_copy_non_overlapping(aligned_mem, old_ptr, min(old_size, size)) - aligned_free(old_ptr) - } - - return byte_slice(aligned_mem, size), nil - } - - aligned_free :: proc(p: rawptr) { - if p != nil { - heap_free(([^]rawptr)(p)[-1]) - } - } - - aligned_resize :: proc(p: rawptr, old_size: int, new_size: int, new_alignment: int, zero_memory := true) -> (new_memory: []byte, err: Allocator_Error) { - if p == nil { - return aligned_alloc(new_size, new_alignment, nil, old_size, zero_memory) - } - - new_memory = aligned_alloc(new_size, new_alignment, p, old_size, zero_memory) or_return - - // NOTE: heap_resize does not zero the new memory, so we do it - if zero_memory && new_size > old_size { - new_region := raw_data(new_memory[old_size:]) - intrinsics.mem_zero(new_region, new_size - old_size) - } - return - } + // Because malloc does not support alignment requests, and aligned_alloc + // has specific requirements for what sizes it supports, this allocator + // over-allocates by the alignment requested and stores the original + // pointer behind the address returned to the user. switch mode { case .Alloc, .Alloc_Non_Zeroed: - return aligned_alloc(size, alignment, nil, 0, mode == .Alloc) + padding := max(alignment, size_of(rawptr)) + ptr := heap_alloc(size + padding, mode == .Alloc) + if ptr == nil { + return nil, .Out_Of_Memory + } + shift := uintptr(padding) - uintptr(ptr) & uintptr(padding-1) + aligned_ptr := rawptr(uintptr(ptr) + shift) + ([^]rawptr)(aligned_ptr)[-1] = ptr + return byte_slice(aligned_ptr, size), nil case .Free: - aligned_free(old_memory) + if old_memory != nil { + heap_free(([^]rawptr)(old_memory)[-1]) + } case .Free_All: return nil, .Mode_Not_Implemented case .Resize, .Resize_Non_Zeroed: - return aligned_resize(old_memory, old_size, size, alignment, mode == .Resize) + new_padding := max(alignment, size_of(rawptr)) + original_ptr := ([^]rawptr)(old_memory)[-1] + ptr: rawptr + + if alignment > align_of(rawptr) { + // The alignment is in excess of what malloc/realloc will return + // for address alignment per the C standard, so we must reallocate + // manually in order to guarantee alignment for the user. + // + // Resizing through realloc simply won't work because it's possible + // that our target address originally only needed a padding of 8 + // bytes, but if we expand the memory used and the address is moved, + // we may then need 16 bytes for proper alignment, for example. + // + // We'll copy the old data later. + ptr = heap_alloc(size + new_padding, mode == .Resize) + + } else { + real_old_size := size_of(rawptr) + old_size + real_new_size := new_padding + size + + ptr = heap_resize(original_ptr, real_old_size, real_new_size, mode == .Resize) + } + + shift := uintptr(new_padding) - uintptr(ptr) & uintptr(new_padding-1) + aligned_ptr := rawptr(uintptr(ptr) + shift) + ([^]rawptr)(aligned_ptr)[-1] = ptr + + if alignment > align_of(rawptr) { + intrinsics.mem_copy_non_overlapping(aligned_ptr, old_memory, min(size, old_size)) + heap_free(original_ptr) + } + + return byte_slice(aligned_ptr, size), nil case .Query_Features: set := (^Allocator_Mode_Set)(old_memory) diff --git a/base/runtime/heap_allocator_implementation.odin b/base/runtime/heap_allocator_implementation.odin index 3faacbe31..5a940688c 100644 --- a/base/runtime/heap_allocator_implementation.odin +++ b/base/runtime/heap_allocator_implementation.odin @@ -1347,7 +1347,7 @@ heap_orphanage_count: int Allocate an arbitrary amount of memory from the heap and optionally zero it. */ @(require_results) -heap_alloc :: proc "contextless" (size: int, zero: bool = true) -> (ptr: rawptr) { +heap_alloc :: proc "contextless" (size: int, zero_memory: bool = true) -> (ptr: rawptr) { assert_contextless(size >= 0, "The heap allocator was given a negative size.") // Handle Huge allocations. @@ -1491,7 +1491,7 @@ heap_alloc :: proc "contextless" (size: int, zero: bool = true) -> (ptr: rawptr) slab.local_free[sector] &~= (1 << index) // Zero the memory, if needed. - if zero && index < uintptr(slab.dirty_bins) { + if zero_memory && index < uintptr(slab.dirty_bins) { // Ensure that the memory zeroing is not optimized out by the compiler. intrinsics.mem_zero_volatile(ptr, rounded_size) // NOTE: A full memory fence should not be needed for any newly-zeroed @@ -1694,7 +1694,7 @@ heap_free :: proc "contextless" (ptr: rawptr) { Resize memory returned by `heap_alloc`. */ @(require_results) -heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int, zero: bool = true) -> (new_ptr: rawptr) { +heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int, zero_memory: bool = true) -> (new_ptr: rawptr) { Size_Category :: enum { Unknown, Bin, @@ -1722,7 +1722,7 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int if new_category != old_category { // A change in size category cannot be optimized. - new_ptr = heap_alloc(new_size, zero) + new_ptr = heap_alloc(new_size, zero_memory) intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) heap_free(old_ptr) heap_debug_cover(.Resize_Crossed_Size_Categories) @@ -1750,7 +1750,7 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int u := uintptr(resized_superpage) + HEAP_HUGE_ALLOCATION_BOOK_KEEPING new_ptr = rawptr(u - u & (HEAP_MAX_ALIGNMENT-1)) - if zero && new_size > old_size { + if zero_memory && new_size > old_size { intrinsics.mem_zero_volatile( rawptr(uintptr(new_ptr) + uintptr(old_size)), new_size - old_size, @@ -1771,7 +1771,7 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int contiguous_new := heap_slabs_needed_for_size(new_size) if contiguous_new == contiguous_old { // We already have enough slabs to serve the request. - if zero && new_size > old_size { + if zero_memory && new_size > old_size { intrinsics.mem_zero_volatile( rawptr(uintptr(old_ptr) + uintptr(old_size)), new_size - old_size, @@ -1785,7 +1785,7 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int if slab.index + contiguous_new >= HEAP_SLAB_COUNT { // Expanding this slab would go beyond the Superpage. // We need more memory. - new_ptr = heap_alloc(new_size, zero) + new_ptr = heap_alloc(new_size, zero_memory) intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) heap_free(old_ptr) return @@ -1797,7 +1797,7 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int // involve touching the Superpage. // // We must re-allocate. - new_ptr = heap_alloc(new_size, zero) + new_ptr = heap_alloc(new_size, zero_memory) intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) heap_free(old_ptr) heap_debug_cover(.Resize_Wide_Slab_From_Remote_Thread) @@ -1826,7 +1826,7 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int for i := slab.index + contiguous_old; i < slab.index + contiguous_new; i += 1 { if heap_superpage_index_slab(superpage, i).bin_size != 0 { // Contiguous space is unavailable. - new_ptr = heap_alloc(new_size, zero) + new_ptr = heap_alloc(new_size, zero_memory) intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) heap_free(old_ptr) heap_debug_cover(.Resize_Wide_Slab_Failed_To_Find_Contiguous_Expansion) @@ -1859,7 +1859,7 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int // See if a bin rank change is needed. new_rounded_size := heap_round_to_bin_size(new_size) if slab.bin_size == new_rounded_size { - if zero && new_size > old_size { + if zero_memory && new_size > old_size { intrinsics.mem_zero_volatile( rawptr(uintptr(old_ptr) + uintptr(old_size)), new_size - old_size, @@ -1879,7 +1879,7 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int } // Allocate and copy, as a last resort. - new_ptr = heap_alloc(new_size, zero) + new_ptr = heap_alloc(new_size, zero_memory) intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) heap_free(old_ptr) return diff --git a/base/runtime/wasm_allocator.odin b/base/runtime/wasm_allocator.odin index 574f3dd06..c4799fcbe 100644 --- a/base/runtime/wasm_allocator.odin +++ b/base/runtime/wasm_allocator.odin @@ -1,3 +1,4 @@ +#+build !orca #+build wasm32, wasm64p32 package runtime @@ -869,3 +870,22 @@ aligned_realloc :: proc(a: ^WASM_Allocator, ptr: rawptr, alignment, size: uint, return newptr } + +heap_allocator :: default_wasm_allocator +heap_allocator_proc :: wasm_allocator_proc + +@(require_results) +heap_alloc :: proc(size: int, zero_memory: bool = true) -> (ptr: rawptr) { + bytes, _ := wasm_allocator_proc(&global_default_wasm_allocator_data, .Alloc if zero_memory else .Alloc_Non_Zeroed, size, 0, nil, 0) + return raw_data(bytes) +} + +@(require_results) +heap_resize :: proc(old_ptr: rawptr, old_size: int, new_size: int, zero_memory: bool = true) -> (new_ptr: rawptr) { + bytes, _ := wasm_allocator_proc(&global_default_wasm_allocator_data, .Resize if zero_memory else .Resize_Non_Zeroed, new_size, 0, old_ptr, old_size) + return raw_data(bytes) +} + +heap_free :: proc(ptr: rawptr) { + wasm_allocator_proc(&global_default_wasm_allocator_data, .Free, 0, 0, ptr, 0) +}