From 0e9cd0fb6abab5b0ac8ef8328423594ebd8f2060 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 13 Apr 2025 15:20:53 +0200 Subject: [PATCH 1/6] Prepare for `tlsf.free_all` --- core/mem/tlsf/tlsf.odin | 23 ++++++++++++++++------- core/mem/tlsf/tlsf_internal.odin | 5 ++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/mem/tlsf/tlsf.odin b/core/mem/tlsf/tlsf.odin index 8ec5f52b7..a525f5113 100644 --- a/core/mem/tlsf/tlsf.odin +++ b/core/mem/tlsf/tlsf.odin @@ -39,12 +39,14 @@ Allocator :: struct { // statistics like how much memory is still available, // fragmentation, etc. pool: Pool, + + // If we're expected to grow when we run out of memory, + // how much should we ask the backing allocator for? + new_pool_size: int, + } #assert(size_of(Allocator) % ALIGN_SIZE == 0) - - - @(require_results) allocator :: proc(t: ^Allocator) -> runtime.Allocator { return runtime.Allocator{ @@ -60,13 +62,18 @@ init_from_buffer :: proc(control: ^Allocator, buf: []byte) -> Error { return .Invalid_Alignment } - pool_bytes := align_down(len(buf) - POOL_OVERHEAD, ALIGN_SIZE) + pool_bytes := align_down(len(buf), ALIGN_SIZE) - INITIAL_POOL_OVERHEAD if pool_bytes < BLOCK_SIZE_MIN { return .Backing_Buffer_Too_Small } else if pool_bytes > BLOCK_SIZE_MAX { return .Backing_Buffer_Too_Large } + control.pool = Pool{ + data = buf, + allocator = {}, + } + clear(control) return pool_add(control, buf[:]) } @@ -74,7 +81,7 @@ init_from_buffer :: proc(control: ^Allocator, buf: []byte) -> Error { @(require_results) init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, initial_pool_size: int, new_pool_size := 0) -> Error { assert(control != nil) - pool_bytes := align_up(uint(initial_pool_size) + POOL_OVERHEAD, ALIGN_SIZE) + pool_bytes := align_up(uint(initial_pool_size), ALIGN_SIZE) + INITIAL_POOL_OVERHEAD if pool_bytes < BLOCK_SIZE_MIN { return .Backing_Buffer_Too_Small } else if pool_bytes > BLOCK_SIZE_MAX { @@ -85,12 +92,14 @@ init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, ini if backing_err != nil { return .Backing_Allocator_Error } - err := init_from_buffer(control, buf) + control.pool = Pool{ data = buf, allocator = backing, } - return err + + clear(control) + return pool_add(control, buf[:]) } init :: proc{init_from_buffer, init_from_allocator} diff --git a/core/mem/tlsf/tlsf_internal.odin b/core/mem/tlsf/tlsf_internal.odin index 66141fcbb..1f04515bf 100644 --- a/core/mem/tlsf/tlsf_internal.odin +++ b/core/mem/tlsf/tlsf_internal.odin @@ -12,7 +12,6 @@ package mem_tlsf import "base:intrinsics" import "base:runtime" -// import "core:fmt" // log2 of number of linear subdivisions of block sizes. // Larger values require more memory in the control structure. @@ -58,7 +57,6 @@ Pool :: struct { next: ^Pool, } - /* Block header structure. @@ -95,6 +93,7 @@ The `prev_phys_block` field is stored *inside* the previous free block. BLOCK_HEADER_OVERHEAD :: uint(size_of(uint)) POOL_OVERHEAD :: 2 * BLOCK_HEADER_OVERHEAD +INITIAL_POOL_OVERHEAD :: 48 // User data starts directly after the size field in a used block. BLOCK_START_OFFSET :: offset_of(Block_Header, size) + size_of(Block_Header{}.size) @@ -114,7 +113,7 @@ BLOCK_SIZE_MAX :: uint(1) << FL_INDEX_MAX queries using bitmasks and architecture-specific bit-manipulation routines. - NOTE: TLSF spec relies on ffs/fls returning value 0..31. + NOTE: TLSF spec relies on ffs/fls returning a value in the range 0..31. */ @(require_results) From 7088284ff43d0c69553a4619b37dfdf85616cf45 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Mon, 14 Apr 2025 17:13:27 +0200 Subject: [PATCH 2/6] Refactor `core:mem/tlsf`, add `free_all` support. TODO: Allow the TLSF allocator to add additional pools when it would ordinarily OOM by calling its backing allocator. --- core/mem/tlsf/tlsf.odin | 40 +- core/mem/tlsf/tlsf_internal.odin | 872 +++++++++++++++--------------- tests/core/mem/test_core_mem.odin | 102 ++++ 3 files changed, 569 insertions(+), 445 deletions(-) diff --git a/core/mem/tlsf/tlsf.odin b/core/mem/tlsf/tlsf.odin index a525f5113..e5ab95947 100644 --- a/core/mem/tlsf/tlsf.odin +++ b/core/mem/tlsf/tlsf.odin @@ -10,6 +10,7 @@ // package mem_tlsf implements a Two Level Segregated Fit memory allocator. package mem_tlsf +import "base:intrinsics" import "base:runtime" Error :: enum byte { @@ -42,7 +43,7 @@ Allocator :: struct { // If we're expected to grow when we run out of memory, // how much should we ask the backing allocator for? - new_pool_size: int, + new_pool_size: uint, } #assert(size_of(Allocator) % ALIGN_SIZE == 0) @@ -74,12 +75,11 @@ init_from_buffer :: proc(control: ^Allocator, buf: []byte) -> Error { allocator = {}, } - clear(control) - return pool_add(control, buf[:]) + return free_all(control) } @(require_results) -init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, initial_pool_size: int, new_pool_size := 0) -> Error { +init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, initial_pool_size: int) -> Error { assert(control != nil) pool_bytes := align_up(uint(initial_pool_size), ALIGN_SIZE) + INITIAL_POOL_OVERHEAD if pool_bytes < BLOCK_SIZE_MIN { @@ -98,8 +98,9 @@ init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, ini allocator = backing, } - clear(control) - return pool_add(control, buf[:]) + // TODO(Jeroen): Add automatically growing the pools from the backing allocator + + return free_all(control) } init :: proc{init_from_buffer, init_from_allocator} @@ -112,8 +113,6 @@ destroy :: proc(control: ^Allocator) { // No need to call `pool_remove` or anything, as they're they're embedded in the backing memory. // We do however need to free the `Pool` tracking entities and the backing memory itself. - // As `Allocator` is embedded in the first backing slice, the `control` pointer will be - // invalid after this call. for p := control.pool.next; p != nil; { next := p.next @@ -145,9 +144,8 @@ allocator_proc :: proc(allocator_data: rawptr, mode: runtime.Allocator_Mode, return nil, nil case .Free_All: - // NOTE: this doesn't work right at the moment, Jeroen has it on his to-do list :) - // clear(control) - return nil, .Mode_Not_Implemented + free_all(control) + return nil, nil case .Resize: return resize(control, old_memory, uint(old_size), uint(size), uint(alignment)) @@ -168,3 +166,23 @@ allocator_proc :: proc(allocator_data: rawptr, mode: runtime.Allocator_Mode, return nil, nil } + +// Exported solely to facilitate testing +@(require_results) +ffs :: proc "contextless" (word: u32) -> (bit: i32) { + return -1 if word == 0 else i32(intrinsics.count_trailing_zeros(word)) +} + +// Exported solely to facilitate testing +@(require_results) +fls :: proc "contextless" (word: u32) -> (bit: i32) { + N :: (size_of(u32) * 8) - 1 + return i32(N - intrinsics.count_leading_zeros(word)) +} + +// Exported solely to facilitate testing +@(require_results) +fls_uint :: proc "contextless" (size: uint) -> (bit: i32) { + N :: (size_of(uint) * 8) - 1 + return i32(N - intrinsics.count_leading_zeros(size)) +} \ No newline at end of file diff --git a/core/mem/tlsf/tlsf_internal.odin b/core/mem/tlsf/tlsf_internal.odin index 1f04515bf..53c658f90 100644 --- a/core/mem/tlsf/tlsf_internal.odin +++ b/core/mem/tlsf/tlsf_internal.odin @@ -7,7 +7,6 @@ Jeroen van Rijn: Source port */ - package mem_tlsf import "base:intrinsics" @@ -106,436 +105,12 @@ bits for `FL_INDEX`. BLOCK_SIZE_MIN :: uint(size_of(Block_Header) - size_of(^Block_Header)) BLOCK_SIZE_MAX :: uint(1) << FL_INDEX_MAX -/* - TLSF achieves O(1) cost for `alloc` and `free` operations by limiting - the search for a free block to a free list of guaranteed size - adequate to fulfill the request, combined with efficient free list - queries using bitmasks and architecture-specific bit-manipulation - routines. - - NOTE: TLSF spec relies on ffs/fls returning a value in the range 0..31. -*/ - -@(require_results) -ffs :: proc "contextless" (word: u32) -> (bit: i32) { - return -1 if word == 0 else i32(intrinsics.count_trailing_zeros(word)) -} - -@(require_results) -fls :: proc "contextless" (word: u32) -> (bit: i32) { - N :: (size_of(u32) * 8) - 1 - return i32(N - intrinsics.count_leading_zeros(word)) -} - -@(require_results) -fls_uint :: proc "contextless" (size: uint) -> (bit: i32) { - N :: (size_of(uint) * 8) - 1 - return i32(N - intrinsics.count_leading_zeros(size)) -} - -@(require_results) -block_size :: proc "contextless" (block: ^Block_Header) -> (size: uint) { - return block.size &~ (BLOCK_HEADER_FREE | BLOCK_HEADER_PREV_FREE) -} - -block_set_size :: proc "contextless" (block: ^Block_Header, size: uint) { - old_size := block.size - block.size = size | (old_size & (BLOCK_HEADER_FREE | BLOCK_HEADER_PREV_FREE)) -} - -@(require_results) -block_is_last :: proc "contextless" (block: ^Block_Header) -> (is_last: bool) { - return block_size(block) == 0 -} - -@(require_results) -block_is_free :: proc "contextless" (block: ^Block_Header) -> (is_free: bool) { - return (block.size & BLOCK_HEADER_FREE) == BLOCK_HEADER_FREE -} - -block_set_free :: proc "contextless" (block: ^Block_Header) { - block.size |= BLOCK_HEADER_FREE -} - -block_set_used :: proc "contextless" (block: ^Block_Header) { - block.size &~= BLOCK_HEADER_FREE -} - -@(require_results) -block_is_prev_free :: proc "contextless" (block: ^Block_Header) -> (is_prev_free: bool) { - return (block.size & BLOCK_HEADER_PREV_FREE) == BLOCK_HEADER_PREV_FREE -} - -block_set_prev_free :: proc "contextless" (block: ^Block_Header) { - block.size |= BLOCK_HEADER_PREV_FREE -} - -block_set_prev_used :: proc "contextless" (block: ^Block_Header) { - block.size &~= BLOCK_HEADER_PREV_FREE -} - -@(require_results) -block_from_ptr :: proc(ptr: rawptr) -> (block_ptr: ^Block_Header) { - return (^Block_Header)(uintptr(ptr) - BLOCK_START_OFFSET) -} - -@(require_results) -block_to_ptr :: proc(block: ^Block_Header) -> (ptr: rawptr) { - return rawptr(uintptr(block) + BLOCK_START_OFFSET) -} - -// Return location of next block after block of given size. -@(require_results) -offset_to_block :: proc(ptr: rawptr, size: uint) -> (block: ^Block_Header) { - return (^Block_Header)(uintptr(ptr) + uintptr(size)) -} - -@(require_results) -offset_to_block_backwards :: proc(ptr: rawptr, size: uint) -> (block: ^Block_Header) { - return (^Block_Header)(uintptr(ptr) - uintptr(size)) -} - -// Return location of previous block. -@(require_results) -block_prev :: proc(block: ^Block_Header) -> (prev: ^Block_Header) { - assert(block_is_prev_free(block), "previous block must be free") - return block.prev_phys_block -} - -// Return location of next existing block. -@(require_results) -block_next :: proc(block: ^Block_Header) -> (next: ^Block_Header) { - return offset_to_block(block_to_ptr(block), block_size(block) - BLOCK_HEADER_OVERHEAD) -} - -// Link a new block with its physical neighbor, return the neighbor. -@(require_results) -block_link_next :: proc(block: ^Block_Header) -> (next: ^Block_Header) { - next = block_next(block) - next.prev_phys_block = block - return -} - -block_mark_as_free :: proc(block: ^Block_Header) { - // Link the block to the next block, first. - next := block_link_next(block) - block_set_prev_free(next) - block_set_free(block) -} - -block_mark_as_used :: proc(block: ^Block_Header) { - next := block_next(block) - block_set_prev_used(next) - block_set_used(block) -} - -@(require_results) -align_up :: proc(x, align: uint) -> (aligned: uint) { - assert(0 == (align & (align - 1)), "must align to a power of two") - return (x + (align - 1)) &~ (align - 1) -} - -@(require_results) -align_down :: proc(x, align: uint) -> (aligned: uint) { - assert(0 == (align & (align - 1)), "must align to a power of two") - return x - (x & (align - 1)) -} - -@(require_results) -align_ptr :: proc(ptr: rawptr, align: uint) -> (aligned: rawptr) { - assert(0 == (align & (align - 1)), "must align to a power of two") - align_mask := uintptr(align) - 1 - _ptr := uintptr(ptr) - _aligned := (_ptr + align_mask) &~ (align_mask) - return rawptr(_aligned) -} - -// Adjust an allocation size to be aligned to word size, and no smaller than internal minimum. -@(require_results) -adjust_request_size :: proc(size, align: uint) -> (adjusted: uint) { - if size == 0 { - return 0 - } - - // aligned size must not exceed `BLOCK_SIZE_MAX`, or we'll go out of bounds on `sl_bitmap`. - if aligned := align_up(size, align); aligned < BLOCK_SIZE_MAX { - adjusted = max(aligned, BLOCK_SIZE_MIN) - } - return -} - -// Adjust an allocation size to be aligned to word size, and no smaller than internal minimum. -@(require_results) -adjust_request_size_with_err :: proc(size, align: uint) -> (adjusted: uint, err: runtime.Allocator_Error) { - if size == 0 { - return 0, nil - } - - // aligned size must not exceed `BLOCK_SIZE_MAX`, or we'll go out of bounds on `sl_bitmap`. - if aligned := align_up(size, align); aligned < BLOCK_SIZE_MAX { - adjusted = min(aligned, BLOCK_SIZE_MAX) - } else { - err = .Out_Of_Memory - } - return -} - -// TLSF utility functions. In most cases these are direct translations of -// the documentation in the research paper. - -@(optimization_mode="favor_size", require_results) -mapping_insert :: proc(size: uint) -> (fl, sl: i32) { - if size < SMALL_BLOCK_SIZE { - // Store small blocks in first list. - sl = i32(size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT) - } else { - fl = fls_uint(size) - sl = i32(size >> (uint(fl) - TLSF_SL_INDEX_COUNT_LOG2)) ~ (1 << TLSF_SL_INDEX_COUNT_LOG2) - fl -= (FL_INDEX_SHIFT - 1) - } - return -} - -@(optimization_mode="favor_size", require_results) -mapping_round :: #force_inline proc(size: uint) -> (rounded: uint) { - rounded = size - if size >= SMALL_BLOCK_SIZE { - round := uint(1 << (uint(fls_uint(size) - TLSF_SL_INDEX_COUNT_LOG2))) - 1 - rounded += round - } - return -} - -// This version rounds up to the next block size (for allocations) -@(optimization_mode="favor_size", require_results) -mapping_search :: proc(size: uint) -> (fl, sl: i32) { - return mapping_insert(mapping_round(size)) -} - -@(require_results) -search_suitable_block :: proc(control: ^Allocator, fli, sli: ^i32) -> (block: ^Block_Header) { - // First, search for a block in the list associated with the given fl/sl index. - fl := fli^; sl := sli^ - - sl_map := control.sl_bitmap[fli^] & (~u32(0) << uint(sl)) - if sl_map == 0 { - // No block exists. Search in the next largest first-level list. - fl_map := control.fl_bitmap & (~u32(0) << uint(fl + 1)) - if fl_map == 0 { - // No free blocks available, memory has been exhausted. - return {} - } - - fl = ffs(fl_map) - fli^ = fl - sl_map = control.sl_bitmap[fl] - } - assert(sl_map != 0, "internal error - second level bitmap is null") - sl = ffs(sl_map) - sli^ = sl - - // Return the first block in the free list. - return control.blocks[fl][sl] -} - -// Remove a free block from the free list. -remove_free_block :: proc(control: ^Allocator, block: ^Block_Header, fl: i32, sl: i32) { - prev := block.prev_free - next := block.next_free - assert(prev != nil, "prev_free can not be nil") - assert(next != nil, "next_free can not be nil") - next.prev_free = prev - prev.next_free = next - - // If this block is the head of the free list, set new head. - if control.blocks[fl][sl] == block { - control.blocks[fl][sl] = next - - // If the new head is nil, clear the bitmap - if next == &control.block_null { - control.sl_bitmap[fl] &~= (u32(1) << uint(sl)) - - // If the second bitmap is now empty, clear the fl bitmap - if control.sl_bitmap[fl] == 0 { - control.fl_bitmap &~= (u32(1) << uint(fl)) - } - } - } -} - -// Insert a free block into the free block list. -insert_free_block :: proc(control: ^Allocator, block: ^Block_Header, fl: i32, sl: i32) { - current := control.blocks[fl][sl] - assert(current != nil, "free lists cannot have a nil entry") - assert(block != nil, "cannot insert a nil entry into the free list") - block.next_free = current - block.prev_free = &control.block_null - current.prev_free = block - - assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE), "block not properly aligned") - - // Insert the new block at the head of the list, and mark the first- and second-level bitmaps appropriately. - control.blocks[fl][sl] = block - control.fl_bitmap |= (u32(1) << uint(fl)) - control.sl_bitmap[fl] |= (u32(1) << uint(sl)) -} - -// Remove a given block from the free list. -block_remove :: proc(control: ^Allocator, block: ^Block_Header) { - fl, sl := mapping_insert(block_size(block)) - remove_free_block(control, block, fl, sl) -} - -// Insert a given block into the free list. -block_insert :: proc(control: ^Allocator, block: ^Block_Header) { - fl, sl := mapping_insert(block_size(block)) - insert_free_block(control, block, fl, sl) -} - -@(require_results) -block_can_split :: proc(block: ^Block_Header, size: uint) -> (can_split: bool) { - return block_size(block) >= size_of(Block_Header) + size -} - -// Split a block into two, the second of which is free. -@(require_results) -block_split :: proc(block: ^Block_Header, size: uint) -> (remaining: ^Block_Header) { - // Calculate the amount of space left in the remaining block. - remaining = offset_to_block(block_to_ptr(block), size - BLOCK_HEADER_OVERHEAD) - - remain_size := block_size(block) - (size + BLOCK_HEADER_OVERHEAD) - - assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE), - "remaining block not aligned properly") - - assert(block_size(block) == remain_size + size + BLOCK_HEADER_OVERHEAD) - block_set_size(remaining, remain_size) - assert(block_size(remaining) >= BLOCK_SIZE_MIN, "block split with invalid size") - - block_set_size(block, size) - block_mark_as_free(remaining) - - return remaining -} - -// Absorb a free block's storage into an adjacent previous free block. -@(require_results) -block_absorb :: proc(prev: ^Block_Header, block: ^Block_Header) -> (absorbed: ^Block_Header) { - assert(!block_is_last(prev), "previous block can't be last") - // Note: Leaves flags untouched. - prev.size += block_size(block) + BLOCK_HEADER_OVERHEAD - _ = block_link_next(prev) - return prev -} - -// Merge a just-freed block with an adjacent previous free block. -@(require_results) -block_merge_prev :: proc(control: ^Allocator, block: ^Block_Header) -> (merged: ^Block_Header) { - merged = block - if (block_is_prev_free(block)) { - prev := block_prev(block) - assert(prev != nil, "prev physical block can't be nil") - assert(block_is_free(prev), "prev block is not free though marked as such") - block_remove(control, prev) - merged = block_absorb(prev, block) - } - return merged -} - -// Merge a just-freed block with an adjacent free block. -@(require_results) -block_merge_next :: proc(control: ^Allocator, block: ^Block_Header) -> (merged: ^Block_Header) { - merged = block - next := block_next(block) - assert(next != nil, "next physical block can't be nil") - - if (block_is_free(next)) { - assert(!block_is_last(block), "previous block can't be last") - block_remove(control, next) - merged = block_absorb(block, next) - } - return merged -} - -// Trim any trailing block space off the end of a free block, return to pool. -block_trim_free :: proc(control: ^Allocator, block: ^Block_Header, size: uint) { - assert(block_is_free(block), "block must be free") - if (block_can_split(block, size)) { - remaining_block := block_split(block, size) - _ = block_link_next(block) - block_set_prev_free(remaining_block) - block_insert(control, remaining_block) - } -} - -// Trim any trailing block space off the end of a used block, return to pool. -block_trim_used :: proc(control: ^Allocator, block: ^Block_Header, size: uint) { - assert(!block_is_free(block), "Block must be used") - if (block_can_split(block, size)) { - // If the next block is free, we must coalesce. - remaining_block := block_split(block, size) - block_set_prev_used(remaining_block) - - remaining_block = block_merge_next(control, remaining_block) - block_insert(control, remaining_block) - } -} - -// Trim leading block space, return to pool. -@(require_results) -block_trim_free_leading :: proc(control: ^Allocator, block: ^Block_Header, size: uint) -> (remaining: ^Block_Header) { - remaining = block - if block_can_split(block, size) { - // We want the 2nd block. - remaining = block_split(block, size - BLOCK_HEADER_OVERHEAD) - block_set_prev_free(remaining) - - _ = block_link_next(block) - block_insert(control, block) - } - return remaining -} - -@(require_results) -block_locate_free :: proc(control: ^Allocator, size: uint) -> (block: ^Block_Header) { - fl, sl: i32 - if size != 0 { - fl, sl = mapping_search(size) - - /* - `mapping_search` can futz with the size, so for excessively large sizes it can sometimes wind up - with indices that are off the end of the block array. So, we protect against that here, - since this is the only call site of `mapping_search`. Note that we don't need to check `sl`, - as it comes from a modulo operation that guarantees it's always in range. - */ - if fl < FL_INDEX_COUNT { - block = search_suitable_block(control, &fl, &sl) - } - } - - if block != nil { - assert(block_size(block) >= size) - remove_free_block(control, block, fl, sl) - } - return block -} - -@(require_results) -block_prepare_used :: proc(control: ^Allocator, block: ^Block_Header, size: uint) -> (res: []byte, err: runtime.Allocator_Error) { - if block != nil { - assert(size != 0, "Size must be non-zero") - block_trim_free(control, block, size) - block_mark_as_used(block) - res = ([^]byte)(block_to_ptr(block))[:size] - } - return -} - // Clear control structure and point all empty lists at the null block -clear :: proc(control: ^Allocator) { +@(private) +free_all :: proc(control: ^Allocator) -> (err: Error) { + // Clear internal structures control.block_null.next_free = &control.block_null control.block_null.prev_free = &control.block_null - control.fl_bitmap = 0 for i in 0.. (err: Error) { assert(uintptr(raw_data(pool)) % ALIGN_SIZE == 0, "Added memory must be aligned") @@ -573,9 +154,11 @@ pool_add :: proc(control: ^Allocator, pool: []u8) -> (err: Error) { block_set_size(next, 0) block_set_used(next) block_set_prev_free(next) + return } +@(private) pool_remove :: proc(control: ^Allocator, pool: []u8) { block := offset_to_block_backwards(raw_data(pool), BLOCK_HEADER_OVERHEAD) @@ -587,7 +170,7 @@ pool_remove :: proc(control: ^Allocator, pool: []u8) { remove_free_block(control, block, fl, sl) } -@(require_results) +@(private, require_results) alloc_bytes_non_zeroed :: proc(control: ^Allocator, size: uint, align: uint) -> (res: []byte, err: runtime.Allocator_Error) { assert(control != nil) adjust := adjust_request_size(size, ALIGN_SIZE) @@ -600,7 +183,7 @@ alloc_bytes_non_zeroed :: proc(control: ^Allocator, size: uint, align: uint) -> return nil, .Out_Of_Memory } - block := block_locate_free(control, aligned_size) + block := block_locate_free(control, aligned_size) if block == nil { return nil, .Out_Of_Memory } @@ -626,7 +209,7 @@ alloc_bytes_non_zeroed :: proc(control: ^Allocator, size: uint, align: uint) -> return block_prepare_used(control, block, adjust) } -@(require_results) +@(private, require_results) alloc_bytes :: proc(control: ^Allocator, size: uint, align: uint) -> (res: []byte, err: runtime.Allocator_Error) { res, err = alloc_bytes_non_zeroed(control, size, align) if err == nil { @@ -652,7 +235,7 @@ free_with_size :: proc(control: ^Allocator, ptr: rawptr, size: uint) { } -@(require_results) +@(private, require_results) resize :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: uint, alignment: uint) -> (res: []byte, err: runtime.Allocator_Error) { assert(control != nil) if ptr != nil && new_size == 0 { @@ -696,7 +279,7 @@ resize :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: uint, align return } -@(require_results) +@(private, require_results) resize_non_zeroed :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: uint, alignment: uint) -> (res: []byte, err: runtime.Allocator_Error) { assert(control != nil) if ptr != nil && new_size == 0 { @@ -735,3 +318,424 @@ resize_non_zeroed :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: res = ([^]byte)(ptr)[:new_size] return } + +/* + TLSF achieves O(1) cost for `alloc` and `free` operations by limiting + the search for a free block to a free list of guaranteed size + adequate to fulfill the request, combined with efficient free list + queries using bitmasks and architecture-specific bit-manipulation + routines. + + NOTE: TLSF spec relies on ffs/fls returning a value in the range 0..31. +*/ + +@(private, require_results) +block_size :: proc "contextless" (block: ^Block_Header) -> (size: uint) { + return block.size &~ (BLOCK_HEADER_FREE | BLOCK_HEADER_PREV_FREE) +} + +@(private) +block_set_size :: proc "contextless" (block: ^Block_Header, size: uint) { + old_size := block.size + block.size = size | (old_size & (BLOCK_HEADER_FREE | BLOCK_HEADER_PREV_FREE)) +} + +@(private, require_results) +block_is_last :: proc "contextless" (block: ^Block_Header) -> (is_last: bool) { + return block_size(block) == 0 +} + +@(private, require_results) +block_is_free :: proc "contextless" (block: ^Block_Header) -> (is_free: bool) { + return (block.size & BLOCK_HEADER_FREE) == BLOCK_HEADER_FREE +} + +@(private) +block_set_free :: proc "contextless" (block: ^Block_Header) { + block.size |= BLOCK_HEADER_FREE +} + +@(private) +block_set_used :: proc "contextless" (block: ^Block_Header) { + block.size &~= BLOCK_HEADER_FREE +} + +@(private, require_results) +block_is_prev_free :: proc "contextless" (block: ^Block_Header) -> (is_prev_free: bool) { + return (block.size & BLOCK_HEADER_PREV_FREE) == BLOCK_HEADER_PREV_FREE +} + +@(private) +block_set_prev_free :: proc "contextless" (block: ^Block_Header) { + block.size |= BLOCK_HEADER_PREV_FREE +} + +@(private) +block_set_prev_used :: proc "contextless" (block: ^Block_Header) { + block.size &~= BLOCK_HEADER_PREV_FREE +} + +@(private, require_results) +block_from_ptr :: proc(ptr: rawptr) -> (block_ptr: ^Block_Header) { + return (^Block_Header)(uintptr(ptr) - BLOCK_START_OFFSET) +} + +@(private, require_results) +block_to_ptr :: proc(block: ^Block_Header) -> (ptr: rawptr) { + return rawptr(uintptr(block) + BLOCK_START_OFFSET) +} + +// Return location of next block after block of given size. +@(private, require_results) +offset_to_block :: proc(ptr: rawptr, size: uint) -> (block: ^Block_Header) { + return (^Block_Header)(uintptr(ptr) + uintptr(size)) +} + +@(private, require_results) +offset_to_block_backwards :: proc(ptr: rawptr, size: uint) -> (block: ^Block_Header) { + return (^Block_Header)(uintptr(ptr) - uintptr(size)) +} + +// Return location of previous block. +@(private, require_results) +block_prev :: proc(block: ^Block_Header) -> (prev: ^Block_Header) { + assert(block_is_prev_free(block), "previous block must be free") + return block.prev_phys_block +} + +// Return location of next existing block. +@(private, require_results) +block_next :: proc(block: ^Block_Header) -> (next: ^Block_Header) { + return offset_to_block(block_to_ptr(block), block_size(block) - BLOCK_HEADER_OVERHEAD) +} + +// Link a new block with its physical neighbor, return the neighbor. +@(private, require_results) +block_link_next :: proc(block: ^Block_Header) -> (next: ^Block_Header) { + next = block_next(block) + next.prev_phys_block = block + return +} + +@(private) +block_mark_as_free :: proc(block: ^Block_Header) { + // Link the block to the next block, first. + next := block_link_next(block) + block_set_prev_free(next) + block_set_free(block) +} + +@(private) +block_mark_as_used :: proc(block: ^Block_Header) { + next := block_next(block) + block_set_prev_used(next) + block_set_used(block) +} + +@(private, require_results) +align_up :: proc(x, align: uint) -> (aligned: uint) { + assert(0 == (align & (align - 1)), "must align to a power of two") + return (x + (align - 1)) &~ (align - 1) +} + +@(private, require_results) +align_down :: proc(x, align: uint) -> (aligned: uint) { + assert(0 == (align & (align - 1)), "must align to a power of two") + return x - (x & (align - 1)) +} + +@(private, require_results) +align_ptr :: proc(ptr: rawptr, align: uint) -> (aligned: rawptr) { + assert(0 == (align & (align - 1)), "must align to a power of two") + align_mask := uintptr(align) - 1 + _ptr := uintptr(ptr) + _aligned := (_ptr + align_mask) &~ (align_mask) + return rawptr(_aligned) +} + +// Adjust an allocation size to be aligned to word size, and no smaller than internal minimum. +@(private, require_results) +adjust_request_size :: proc(size, align: uint) -> (adjusted: uint) { + if size == 0 { + return 0 + } + + // aligned size must not exceed `BLOCK_SIZE_MAX`, or we'll go out of bounds on `sl_bitmap`. + if aligned := align_up(size, align); aligned < BLOCK_SIZE_MAX { + adjusted = max(aligned, BLOCK_SIZE_MIN) + } + return +} + +// Adjust an allocation size to be aligned to word size, and no smaller than internal minimum. +@(private, require_results) +adjust_request_size_with_err :: proc(size, align: uint) -> (adjusted: uint, err: runtime.Allocator_Error) { + if size == 0 { + return 0, nil + } + + // aligned size must not exceed `BLOCK_SIZE_MAX`, or we'll go out of bounds on `sl_bitmap`. + if aligned := align_up(size, align); aligned < BLOCK_SIZE_MAX { + adjusted = min(aligned, BLOCK_SIZE_MAX) + } else { + err = .Out_Of_Memory + } + return +} + +// TLSF utility functions. In most cases these are direct translations of +// the documentation in the research paper. + +@(optimization_mode="favor_size", private, require_results) +mapping_insert :: proc(size: uint) -> (fl, sl: i32) { + if size < SMALL_BLOCK_SIZE { + // Store small blocks in first list. + sl = i32(size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT) + } else { + fl = fls_uint(size) + sl = i32(size >> (uint(fl) - TLSF_SL_INDEX_COUNT_LOG2)) ~ (1 << TLSF_SL_INDEX_COUNT_LOG2) + fl -= (FL_INDEX_SHIFT - 1) + } + return +} + +@(optimization_mode="favor_size", private, require_results) +mapping_round :: #force_inline proc(size: uint) -> (rounded: uint) { + rounded = size + if size >= SMALL_BLOCK_SIZE { + round := uint(1 << (uint(fls_uint(size) - TLSF_SL_INDEX_COUNT_LOG2))) - 1 + rounded += round + } + return +} + +// This version rounds up to the next block size (for allocations) +@(optimization_mode="favor_size", private, require_results) +mapping_search :: proc(size: uint) -> (fl, sl: i32) { + return mapping_insert(mapping_round(size)) +} + +@(private, require_results) +search_suitable_block :: proc(control: ^Allocator, fli, sli: ^i32) -> (block: ^Block_Header) { + // First, search for a block in the list associated with the given fl/sl index. + fl := fli^; sl := sli^ + + sl_map := control.sl_bitmap[fli^] & (~u32(0) << uint(sl)) + if sl_map == 0 { + // No block exists. Search in the next largest first-level list. + fl_map := control.fl_bitmap & (~u32(0) << uint(fl + 1)) + if fl_map == 0 { + // No free blocks available, memory has been exhausted. + return {} + } + + fl = ffs(fl_map) + fli^ = fl + sl_map = control.sl_bitmap[fl] + } + assert(sl_map != 0, "internal error - second level bitmap is null") + sl = ffs(sl_map) + sli^ = sl + + // Return the first block in the free list. + return control.blocks[fl][sl] +} + +// Remove a free block from the free list. +@(private) +remove_free_block :: proc(control: ^Allocator, block: ^Block_Header, fl: i32, sl: i32) { + prev := block.prev_free + next := block.next_free + assert(prev != nil, "prev_free can not be nil") + assert(next != nil, "next_free can not be nil") + next.prev_free = prev + prev.next_free = next + + // If this block is the head of the free list, set new head. + if control.blocks[fl][sl] == block { + control.blocks[fl][sl] = next + + // If the new head is nil, clear the bitmap + if next == &control.block_null { + control.sl_bitmap[fl] &~= (u32(1) << uint(sl)) + + // If the second bitmap is now empty, clear the fl bitmap + if control.sl_bitmap[fl] == 0 { + control.fl_bitmap &~= (u32(1) << uint(fl)) + } + } + } +} + +// Insert a free block into the free block list. +@(private) +insert_free_block :: proc(control: ^Allocator, block: ^Block_Header, fl: i32, sl: i32) { + current := control.blocks[fl][sl] + assert(current != nil, "free lists cannot have a nil entry") + assert(block != nil, "cannot insert a nil entry into the free list") + block.next_free = current + block.prev_free = &control.block_null + current.prev_free = block + + assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE), "block not properly aligned") + + // Insert the new block at the head of the list, and mark the first- and second-level bitmaps appropriately. + control.blocks[fl][sl] = block + control.fl_bitmap |= (u32(1) << uint(fl)) + control.sl_bitmap[fl] |= (u32(1) << uint(sl)) +} + +// Remove a given block from the free list. +@(private) +block_remove :: proc(control: ^Allocator, block: ^Block_Header) { + fl, sl := mapping_insert(block_size(block)) + remove_free_block(control, block, fl, sl) +} + +// Insert a given block into the free list. +@(private) +block_insert :: proc(control: ^Allocator, block: ^Block_Header) { + fl, sl := mapping_insert(block_size(block)) + insert_free_block(control, block, fl, sl) +} + +@(private, require_results) +block_can_split :: proc(block: ^Block_Header, size: uint) -> (can_split: bool) { + return block_size(block) >= size_of(Block_Header) + size +} + +// Split a block into two, the second of which is free. +@(private, require_results) +block_split :: proc(block: ^Block_Header, size: uint) -> (remaining: ^Block_Header) { + // Calculate the amount of space left in the remaining block. + remaining = offset_to_block(block_to_ptr(block), size - BLOCK_HEADER_OVERHEAD) + + remain_size := block_size(block) - (size + BLOCK_HEADER_OVERHEAD) + + assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE), + "remaining block not aligned properly") + + assert(block_size(block) == remain_size + size + BLOCK_HEADER_OVERHEAD) + block_set_size(remaining, remain_size) + assert(block_size(remaining) >= BLOCK_SIZE_MIN, "block split with invalid size") + + block_set_size(block, size) + block_mark_as_free(remaining) + + return remaining +} + +// Absorb a free block's storage into an adjacent previous free block. +@(private, require_results) +block_absorb :: proc(prev: ^Block_Header, block: ^Block_Header) -> (absorbed: ^Block_Header) { + assert(!block_is_last(prev), "previous block can't be last") + // Note: Leaves flags untouched. + prev.size += block_size(block) + BLOCK_HEADER_OVERHEAD + _ = block_link_next(prev) + return prev +} + +// Merge a just-freed block with an adjacent previous free block. +@(private, require_results) +block_merge_prev :: proc(control: ^Allocator, block: ^Block_Header) -> (merged: ^Block_Header) { + merged = block + if (block_is_prev_free(block)) { + prev := block_prev(block) + assert(prev != nil, "prev physical block can't be nil") + assert(block_is_free(prev), "prev block is not free though marked as such") + block_remove(control, prev) + merged = block_absorb(prev, block) + } + return merged +} + +// Merge a just-freed block with an adjacent free block. +@(private, require_results) +block_merge_next :: proc(control: ^Allocator, block: ^Block_Header) -> (merged: ^Block_Header) { + merged = block + next := block_next(block) + assert(next != nil, "next physical block can't be nil") + + if (block_is_free(next)) { + assert(!block_is_last(block), "previous block can't be last") + block_remove(control, next) + merged = block_absorb(block, next) + } + return merged +} + +// Trim any trailing block space off the end of a free block, return to pool. +@(private) +block_trim_free :: proc(control: ^Allocator, block: ^Block_Header, size: uint) { + assert(block_is_free(block), "block must be free") + if (block_can_split(block, size)) { + remaining_block := block_split(block, size) + _ = block_link_next(block) + block_set_prev_free(remaining_block) + block_insert(control, remaining_block) + } +} + +// Trim any trailing block space off the end of a used block, return to pool. +@(private) +block_trim_used :: proc(control: ^Allocator, block: ^Block_Header, size: uint) { + assert(!block_is_free(block), "Block must be used") + if (block_can_split(block, size)) { + // If the next block is free, we must coalesce. + remaining_block := block_split(block, size) + block_set_prev_used(remaining_block) + + remaining_block = block_merge_next(control, remaining_block) + block_insert(control, remaining_block) + } +} + +// Trim leading block space, return to pool. +@(private, require_results) +block_trim_free_leading :: proc(control: ^Allocator, block: ^Block_Header, size: uint) -> (remaining: ^Block_Header) { + remaining = block + if block_can_split(block, size) { + // We want the 2nd block. + remaining = block_split(block, size - BLOCK_HEADER_OVERHEAD) + block_set_prev_free(remaining) + + _ = block_link_next(block) + block_insert(control, block) + } + return remaining +} + +@(private, require_results) +block_locate_free :: proc(control: ^Allocator, size: uint) -> (block: ^Block_Header) { + fl, sl: i32 + if size != 0 { + fl, sl = mapping_search(size) + + /* + `mapping_search` can futz with the size, so for excessively large sizes it can sometimes wind up + with indices that are off the end of the block array. So, we protect against that here, + since this is the only call site of `mapping_search`. Note that we don't need to check `sl`, + as it comes from a modulo operation that guarantees it's always in range. + */ + if fl < FL_INDEX_COUNT { + block = search_suitable_block(control, &fl, &sl) + } + } + + if block != nil { + assert(block_size(block) >= size) + remove_free_block(control, block, fl, sl) + } + return block +} + +@(private, require_results) +block_prepare_used :: proc(control: ^Allocator, block: ^Block_Header, size: uint) -> (res: []byte, err: runtime.Allocator_Error) { + if block != nil { + assert(size != 0, "Size must be non-zero") + block_trim_free(control, block, size) + block_mark_as_used(block) + res = ([^]byte)(block_to_ptr(block))[:size] + } + return +} \ No newline at end of file diff --git a/tests/core/mem/test_core_mem.odin b/tests/core/mem/test_core_mem.odin index 4f2095fca..9717e29e9 100644 --- a/tests/core/mem/test_core_mem.odin +++ b/tests/core/mem/test_core_mem.odin @@ -1,8 +1,10 @@ package test_core_mem +import "core:mem" import "core:mem/tlsf" import "core:mem/virtual" import "core:testing" +import "core:slice" @test test_tlsf_bitscan :: proc(t: ^testing.T) { @@ -54,3 +56,103 @@ test_align_bumping_block_limit :: proc(t: ^testing.T) { testing.expect_value(t, err, nil) testing.expect(t, len(data) == 896) } + +@(test) +tlsf_test_overlap_and_zero :: proc(t: ^testing.T) { + default_allocator := context.allocator + alloc: tlsf.Allocator + defer tlsf.destroy(&alloc) + + NUM_ALLOCATIONS :: 1_000 + BACKING_SIZE :: NUM_ALLOCATIONS * (1_000 + size_of(uintptr)) + + if err := tlsf.init_from_allocator(&alloc, default_allocator, BACKING_SIZE); err != .None { + testing.fail_now(t, "TLSF init error") + } + context.allocator = tlsf.allocator(&alloc) + + allocations := make([dynamic][]byte, 0, NUM_ALLOCATIONS, default_allocator) + defer delete(allocations) + + err: mem.Allocator_Error + s: []byte + + for size := 1; err == .None && size <= NUM_ALLOCATIONS; size += 1 { + s, err = make([]byte, size) + append(&allocations, s) + } + + slice.sort_by(allocations[:len(allocations)], proc(a, b: []byte) -> bool { + return uintptr(raw_data(a)) < uintptr(raw_data((b))) + }) + + for i in 0..= 10) + + free_all(tlsf.allocator(&alloc)) + + for { + s := make([]byte, ALLOCATION_SIZE) or_break + append(&allocations[1], s) + } + testing.expect(t, len(allocations[1]) >= 10) + + for i in 0..= b_end && b_end >= a_start { + testing.fail_now(t, "Allocations overlapped") + } +} From beee27dec06d59898e0e71d12a9d77e9d15d8e61 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Mon, 14 Apr 2025 19:49:55 +0200 Subject: [PATCH 3/6] Allow `core:mem/tlsf` to automatically add new pools. New features: - If TLSF can't service an allocation made on it, and it's initialized with `new_pool_size` > 0, it will ask the backing allocator for additional memory. - `estimate_pool_size` can tell you what size your initial (and `new_pool_size`) ought to be if you want to make `count` allocations of `size` and `alignment`, or in its other form, how much backing memory is needed for `count` allocations of `type` and its corresponding size and alignment. --- core/mem/tlsf/tlsf.odin | 25 ++++++++++++++---- core/mem/tlsf/tlsf_internal.odin | 35 +++++++++++++++++++++++++- tests/core/mem/test_core_mem.odin | 42 ++++++++++++++++++++++++++++++- 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/core/mem/tlsf/tlsf.odin b/core/mem/tlsf/tlsf.odin index e5ab95947..4bfcc22d2 100644 --- a/core/mem/tlsf/tlsf.odin +++ b/core/mem/tlsf/tlsf.odin @@ -22,7 +22,6 @@ Error :: enum byte { Backing_Allocator_Error = 5, } - Allocator :: struct { // Empty lists point at this block to indicate they are free. block_null: Block_Header, @@ -44,7 +43,6 @@ Allocator :: struct { // If we're expected to grow when we run out of memory, // how much should we ask the backing allocator for? new_pool_size: uint, - } #assert(size_of(Allocator) % ALIGN_SIZE == 0) @@ -56,6 +54,21 @@ allocator :: proc(t: ^Allocator) -> runtime.Allocator { } } +// Tries to estimate a pool size sufficient for `count` allocations, each of `size` and with `alignment`. +estimate_pool_from_size_alignment :: proc(count: int, size: int, alignment: int) -> (pool_size: int) { + per_allocation := align_up(uint(size + alignment) + BLOCK_HEADER_OVERHEAD, ALIGN_SIZE) + return count * int(per_allocation) + int(INITIAL_POOL_OVERHEAD) +} + +// Tries to estimate a pool size sufficient for `count` allocations of `type`. +estimate_pool_from_typeid :: proc(count: int, type: typeid) -> (pool_size: int) { + ti := type_info_of(type) + return estimate_pool_size(count, ti.size, ti.align) +} + +estimate_pool_size :: proc{estimate_pool_from_size_alignment, estimate_pool_from_typeid} + + @(require_results) init_from_buffer :: proc(control: ^Allocator, buf: []byte) -> Error { assert(control != nil) @@ -63,7 +76,7 @@ init_from_buffer :: proc(control: ^Allocator, buf: []byte) -> Error { return .Invalid_Alignment } - pool_bytes := align_down(len(buf), ALIGN_SIZE) - INITIAL_POOL_OVERHEAD + pool_bytes := align_down(len(buf) - INITIAL_POOL_OVERHEAD, ALIGN_SIZE) if pool_bytes < BLOCK_SIZE_MIN { return .Backing_Buffer_Too_Small } else if pool_bytes > BLOCK_SIZE_MAX { @@ -79,9 +92,9 @@ init_from_buffer :: proc(control: ^Allocator, buf: []byte) -> Error { } @(require_results) -init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, initial_pool_size: int) -> Error { +init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, initial_pool_size: int, new_pool_size := 0) -> Error { assert(control != nil) - pool_bytes := align_up(uint(initial_pool_size), ALIGN_SIZE) + INITIAL_POOL_OVERHEAD + pool_bytes := uint(estimate_pool_size(1, initial_pool_size, ALIGN_SIZE)) if pool_bytes < BLOCK_SIZE_MIN { return .Backing_Buffer_Too_Small } else if pool_bytes > BLOCK_SIZE_MAX { @@ -98,6 +111,8 @@ init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, ini allocator = backing, } + control.new_pool_size = uint(new_pool_size) + // TODO(Jeroen): Add automatically growing the pools from the backing allocator return free_all(control) diff --git a/core/mem/tlsf/tlsf_internal.odin b/core/mem/tlsf/tlsf_internal.odin index 53c658f90..9aee2097d 100644 --- a/core/mem/tlsf/tlsf_internal.odin +++ b/core/mem/tlsf/tlsf_internal.odin @@ -185,7 +185,40 @@ alloc_bytes_non_zeroed :: proc(control: ^Allocator, size: uint, align: uint) -> block := block_locate_free(control, aligned_size) if block == nil { - return nil, .Out_Of_Memory + // OOM: Couldn't find block of `aligned_size` bytes. + if control.new_pool_size > 0 && control.pool.allocator.procedure != nil { + // TLSF is configured to grow. Trying to allocate a new pool of `control.new_pool_size` bytes. + + new_pool_buf := runtime.make_aligned([]byte, control.new_pool_size, ALIGN_SIZE, control.pool.allocator) or_return + + // Add new pool to control structure + if pool_add_err := pool_add(control, new_pool_buf); pool_add_err != .None { + delete(new_pool_buf, control.pool.allocator) + return nil, .Out_Of_Memory + } + + // Allocate a new link in the `control.pool` tracking structure. + new_pool := new_clone(Pool{ + data = new_pool_buf, + allocator = control.pool.allocator, + next = nil, + }, control.pool.allocator) or_return + + p := &control.pool + for p.next != nil { + p = p.next + } + p.next = new_pool + + // Try again to find free block + block = block_locate_free(control, aligned_size) + if block == nil { + return nil, .Out_Of_Memory + } + } else { + // TLSF is non-growing. We're done. + return nil, .Out_Of_Memory + } } ptr := block_to_ptr(block) aligned := align_ptr(ptr, align) diff --git a/tests/core/mem/test_core_mem.odin b/tests/core/mem/test_core_mem.odin index 9717e29e9..d50d15999 100644 --- a/tests/core/mem/test_core_mem.odin +++ b/tests/core/mem/test_core_mem.odin @@ -82,7 +82,47 @@ tlsf_test_overlap_and_zero :: proc(t: ^testing.T) { append(&allocations, s) } - slice.sort_by(allocations[:len(allocations)], proc(a, b: []byte) -> bool { + slice.sort_by(allocations[:], proc(a, b: []byte) -> bool { + return uintptr(raw_data(a)) < uintptr(raw_data((b))) + }) + + for i in 0.. bool { return uintptr(raw_data(a)) < uintptr(raw_data((b))) }) From 3d13beb3eca4e8f17e751962074a25cdaeb846d1 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Mon, 14 Apr 2025 20:09:55 +0200 Subject: [PATCH 4/6] Remove now-implemented TODO --- core/mem/tlsf/tlsf.odin | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/mem/tlsf/tlsf.odin b/core/mem/tlsf/tlsf.odin index 4bfcc22d2..4ce6e54d9 100644 --- a/core/mem/tlsf/tlsf.odin +++ b/core/mem/tlsf/tlsf.odin @@ -113,8 +113,6 @@ init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, ini control.new_pool_size = uint(new_pool_size) - // TODO(Jeroen): Add automatically growing the pools from the backing allocator - return free_all(control) } init :: proc{init_from_buffer, init_from_allocator} From 0fc141db5de0a635d7f20782e694636af18383f2 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Mon, 14 Apr 2025 20:40:05 +0200 Subject: [PATCH 5/6] `core:mem/tlsf`: Add early-out in OOM logic This implementation doesn't allow for out-of-band allocations to be passed through, as it's not designed to track those. Nor is it able to signal those allocations then need to be freed on the backing allocator, as opposed to regular allocations handled for you when you `destroy` the TLSF instance. So if we're asked for more than we're configured to grow by, we can fail with an OOM error early, without adding a new pool. --- core/mem/tlsf/tlsf_internal.odin | 14 +++++++++++++- tests/core/mem/test_core_mem.odin | 7 ++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/core/mem/tlsf/tlsf_internal.odin b/core/mem/tlsf/tlsf_internal.odin index 9aee2097d..f8a9bf60c 100644 --- a/core/mem/tlsf/tlsf_internal.odin +++ b/core/mem/tlsf/tlsf_internal.odin @@ -187,8 +187,20 @@ alloc_bytes_non_zeroed :: proc(control: ^Allocator, size: uint, align: uint) -> if block == nil { // OOM: Couldn't find block of `aligned_size` bytes. if control.new_pool_size > 0 && control.pool.allocator.procedure != nil { - // TLSF is configured to grow. Trying to allocate a new pool of `control.new_pool_size` bytes. + // TLSF is configured to grow. + /* + This implementation doesn't allow for out-of-band allocations to be passed through, as it's not designed to + track those. Nor is it able to signal those allocations then need to be freed on the backing allocator, + as opposed to regular allocations handled for you when you `destroy` the TLSF instance. + + So if we're asked for more than we're configured to grow by, we can fail with an OOM error early, without adding a new pool. + */ + if aligned_size > control.new_pool_size { + return nil, .Out_Of_Memory + } + + // Trying to allocate a new pool of `control.new_pool_size` bytes. new_pool_buf := runtime.make_aligned([]byte, control.new_pool_size, ALIGN_SIZE, control.pool.allocator) or_return // Add new pool to control structure diff --git a/tests/core/mem/test_core_mem.odin b/tests/core/mem/test_core_mem.odin index d50d15999..bd072b4e9 100644 --- a/tests/core/mem/test_core_mem.odin +++ b/tests/core/mem/test_core_mem.odin @@ -111,11 +111,8 @@ tlsf_test_grow_pools :: proc(t: ^testing.T) { } context.allocator = tlsf.allocator(&alloc) - err: mem.Allocator_Error - s: []byte - - for err == .None && len(allocations) < NUM_ALLOCATIONS { - s, err = make([]byte, ALLOC_SIZE) + for len(allocations) < NUM_ALLOCATIONS { + s := make([]byte, ALLOC_SIZE) or_break testing.expect_value(t, len(s), ALLOC_SIZE) append(&allocations, s) } From 11da543f26fff002da8281757f141f6a5f690666 Mon Sep 17 00:00:00 2001 From: Laytan Date: Mon, 14 Apr 2025 21:15:58 +0200 Subject: [PATCH 6/6] net: add ECONNRESET to the error handling of recv --- core/net/errors_linux.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/net/errors_linux.odin b/core/net/errors_linux.odin index da9811d71..237579f28 100644 --- a/core/net/errors_linux.odin +++ b/core/net/errors_linux.odin @@ -159,7 +159,7 @@ _tcp_recv_error :: proc(errno: linux.Errno) -> TCP_Recv_Error { return .Invalid_Argument case .ENOTCONN: return .Not_Connected - case .ECONNREFUSED: + case .ECONNREFUSED, .ECONNRESET: return .Connection_Closed case .ETIMEDOUT: return .Timeout @@ -179,7 +179,7 @@ _udp_recv_error :: proc(errno: linux.Errno) -> UDP_Recv_Error { #partial switch errno { case .EBADF, .ENOTSOCK, .EFAULT: return .Invalid_Argument - case .ECONNREFUSED, .ENOTCONN: + case .ECONNREFUSED, .ENOTCONN, .ECONNRESET: return .Connection_Refused case .ETIMEDOUT: return .Timeout