diff --git a/core/mem/tlsf/tlsf.odin b/core/mem/tlsf/tlsf.odin index 8ec5f52b7..4ce6e54d9 100644 --- a/core/mem/tlsf/tlsf.odin +++ b/core/mem/tlsf/tlsf.odin @@ -10,6 +10,7 @@ // package mem_tlsf implements a Two Level Segregated Fit memory allocator. package mem_tlsf +import "base:intrinsics" import "base:runtime" Error :: enum byte { @@ -21,7 +22,6 @@ Error :: enum byte { Backing_Allocator_Error = 5, } - Allocator :: struct { // Empty lists point at this block to indicate they are free. block_null: Block_Header, @@ -39,12 +39,13 @@ Allocator :: struct { // statistics like how much memory is still available, // fragmentation, etc. pool: Pool, + + // If we're expected to grow when we run out of memory, + // how much should we ask the backing allocator for? + new_pool_size: uint, } #assert(size_of(Allocator) % ALIGN_SIZE == 0) - - - @(require_results) allocator :: proc(t: ^Allocator) -> runtime.Allocator { return runtime.Allocator{ @@ -53,6 +54,21 @@ allocator :: proc(t: ^Allocator) -> runtime.Allocator { } } +// Tries to estimate a pool size sufficient for `count` allocations, each of `size` and with `alignment`. +estimate_pool_from_size_alignment :: proc(count: int, size: int, alignment: int) -> (pool_size: int) { + per_allocation := align_up(uint(size + alignment) + BLOCK_HEADER_OVERHEAD, ALIGN_SIZE) + return count * int(per_allocation) + int(INITIAL_POOL_OVERHEAD) +} + +// Tries to estimate a pool size sufficient for `count` allocations of `type`. +estimate_pool_from_typeid :: proc(count: int, type: typeid) -> (pool_size: int) { + ti := type_info_of(type) + return estimate_pool_size(count, ti.size, ti.align) +} + +estimate_pool_size :: proc{estimate_pool_from_size_alignment, estimate_pool_from_typeid} + + @(require_results) init_from_buffer :: proc(control: ^Allocator, buf: []byte) -> Error { assert(control != nil) @@ -60,21 +76,25 @@ init_from_buffer :: proc(control: ^Allocator, buf: []byte) -> Error { return .Invalid_Alignment } - pool_bytes := align_down(len(buf) - POOL_OVERHEAD, ALIGN_SIZE) + pool_bytes := align_down(len(buf) - INITIAL_POOL_OVERHEAD, ALIGN_SIZE) if pool_bytes < BLOCK_SIZE_MIN { return .Backing_Buffer_Too_Small } else if pool_bytes > BLOCK_SIZE_MAX { return .Backing_Buffer_Too_Large } - clear(control) - return pool_add(control, buf[:]) + control.pool = Pool{ + data = buf, + allocator = {}, + } + + return free_all(control) } @(require_results) init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, initial_pool_size: int, new_pool_size := 0) -> Error { assert(control != nil) - pool_bytes := align_up(uint(initial_pool_size) + POOL_OVERHEAD, ALIGN_SIZE) + pool_bytes := uint(estimate_pool_size(1, initial_pool_size, ALIGN_SIZE)) if pool_bytes < BLOCK_SIZE_MIN { return .Backing_Buffer_Too_Small } else if pool_bytes > BLOCK_SIZE_MAX { @@ -85,12 +105,15 @@ init_from_allocator :: proc(control: ^Allocator, backing: runtime.Allocator, ini if backing_err != nil { return .Backing_Allocator_Error } - err := init_from_buffer(control, buf) + control.pool = Pool{ data = buf, allocator = backing, } - return err + + control.new_pool_size = uint(new_pool_size) + + return free_all(control) } init :: proc{init_from_buffer, init_from_allocator} @@ -103,8 +126,6 @@ destroy :: proc(control: ^Allocator) { // No need to call `pool_remove` or anything, as they're they're embedded in the backing memory. // We do however need to free the `Pool` tracking entities and the backing memory itself. - // As `Allocator` is embedded in the first backing slice, the `control` pointer will be - // invalid after this call. for p := control.pool.next; p != nil; { next := p.next @@ -136,9 +157,8 @@ allocator_proc :: proc(allocator_data: rawptr, mode: runtime.Allocator_Mode, return nil, nil case .Free_All: - // NOTE: this doesn't work right at the moment, Jeroen has it on his to-do list :) - // clear(control) - return nil, .Mode_Not_Implemented + free_all(control) + return nil, nil case .Resize: return resize(control, old_memory, uint(old_size), uint(size), uint(alignment)) @@ -159,3 +179,23 @@ allocator_proc :: proc(allocator_data: rawptr, mode: runtime.Allocator_Mode, return nil, nil } + +// Exported solely to facilitate testing +@(require_results) +ffs :: proc "contextless" (word: u32) -> (bit: i32) { + return -1 if word == 0 else i32(intrinsics.count_trailing_zeros(word)) +} + +// Exported solely to facilitate testing +@(require_results) +fls :: proc "contextless" (word: u32) -> (bit: i32) { + N :: (size_of(u32) * 8) - 1 + return i32(N - intrinsics.count_leading_zeros(word)) +} + +// Exported solely to facilitate testing +@(require_results) +fls_uint :: proc "contextless" (size: uint) -> (bit: i32) { + N :: (size_of(uint) * 8) - 1 + return i32(N - intrinsics.count_leading_zeros(size)) +} \ No newline at end of file diff --git a/core/mem/tlsf/tlsf_internal.odin b/core/mem/tlsf/tlsf_internal.odin index 66141fcbb..f8a9bf60c 100644 --- a/core/mem/tlsf/tlsf_internal.odin +++ b/core/mem/tlsf/tlsf_internal.odin @@ -7,12 +7,10 @@ Jeroen van Rijn: Source port */ - package mem_tlsf import "base:intrinsics" import "base:runtime" -// import "core:fmt" // log2 of number of linear subdivisions of block sizes. // Larger values require more memory in the control structure. @@ -58,7 +56,6 @@ Pool :: struct { next: ^Pool, } - /* Block header structure. @@ -95,6 +92,7 @@ The `prev_phys_block` field is stored *inside* the previous free block. BLOCK_HEADER_OVERHEAD :: uint(size_of(uint)) POOL_OVERHEAD :: 2 * BLOCK_HEADER_OVERHEAD +INITIAL_POOL_OVERHEAD :: 48 // User data starts directly after the size field in a used block. BLOCK_START_OFFSET :: offset_of(Block_Header, size) + size_of(Block_Header{}.size) @@ -107,436 +105,12 @@ bits for `FL_INDEX`. BLOCK_SIZE_MIN :: uint(size_of(Block_Header) - size_of(^Block_Header)) BLOCK_SIZE_MAX :: uint(1) << FL_INDEX_MAX -/* - TLSF achieves O(1) cost for `alloc` and `free` operations by limiting - the search for a free block to a free list of guaranteed size - adequate to fulfill the request, combined with efficient free list - queries using bitmasks and architecture-specific bit-manipulation - routines. - - NOTE: TLSF spec relies on ffs/fls returning value 0..31. -*/ - -@(require_results) -ffs :: proc "contextless" (word: u32) -> (bit: i32) { - return -1 if word == 0 else i32(intrinsics.count_trailing_zeros(word)) -} - -@(require_results) -fls :: proc "contextless" (word: u32) -> (bit: i32) { - N :: (size_of(u32) * 8) - 1 - return i32(N - intrinsics.count_leading_zeros(word)) -} - -@(require_results) -fls_uint :: proc "contextless" (size: uint) -> (bit: i32) { - N :: (size_of(uint) * 8) - 1 - return i32(N - intrinsics.count_leading_zeros(size)) -} - -@(require_results) -block_size :: proc "contextless" (block: ^Block_Header) -> (size: uint) { - return block.size &~ (BLOCK_HEADER_FREE | BLOCK_HEADER_PREV_FREE) -} - -block_set_size :: proc "contextless" (block: ^Block_Header, size: uint) { - old_size := block.size - block.size = size | (old_size & (BLOCK_HEADER_FREE | BLOCK_HEADER_PREV_FREE)) -} - -@(require_results) -block_is_last :: proc "contextless" (block: ^Block_Header) -> (is_last: bool) { - return block_size(block) == 0 -} - -@(require_results) -block_is_free :: proc "contextless" (block: ^Block_Header) -> (is_free: bool) { - return (block.size & BLOCK_HEADER_FREE) == BLOCK_HEADER_FREE -} - -block_set_free :: proc "contextless" (block: ^Block_Header) { - block.size |= BLOCK_HEADER_FREE -} - -block_set_used :: proc "contextless" (block: ^Block_Header) { - block.size &~= BLOCK_HEADER_FREE -} - -@(require_results) -block_is_prev_free :: proc "contextless" (block: ^Block_Header) -> (is_prev_free: bool) { - return (block.size & BLOCK_HEADER_PREV_FREE) == BLOCK_HEADER_PREV_FREE -} - -block_set_prev_free :: proc "contextless" (block: ^Block_Header) { - block.size |= BLOCK_HEADER_PREV_FREE -} - -block_set_prev_used :: proc "contextless" (block: ^Block_Header) { - block.size &~= BLOCK_HEADER_PREV_FREE -} - -@(require_results) -block_from_ptr :: proc(ptr: rawptr) -> (block_ptr: ^Block_Header) { - return (^Block_Header)(uintptr(ptr) - BLOCK_START_OFFSET) -} - -@(require_results) -block_to_ptr :: proc(block: ^Block_Header) -> (ptr: rawptr) { - return rawptr(uintptr(block) + BLOCK_START_OFFSET) -} - -// Return location of next block after block of given size. -@(require_results) -offset_to_block :: proc(ptr: rawptr, size: uint) -> (block: ^Block_Header) { - return (^Block_Header)(uintptr(ptr) + uintptr(size)) -} - -@(require_results) -offset_to_block_backwards :: proc(ptr: rawptr, size: uint) -> (block: ^Block_Header) { - return (^Block_Header)(uintptr(ptr) - uintptr(size)) -} - -// Return location of previous block. -@(require_results) -block_prev :: proc(block: ^Block_Header) -> (prev: ^Block_Header) { - assert(block_is_prev_free(block), "previous block must be free") - return block.prev_phys_block -} - -// Return location of next existing block. -@(require_results) -block_next :: proc(block: ^Block_Header) -> (next: ^Block_Header) { - return offset_to_block(block_to_ptr(block), block_size(block) - BLOCK_HEADER_OVERHEAD) -} - -// Link a new block with its physical neighbor, return the neighbor. -@(require_results) -block_link_next :: proc(block: ^Block_Header) -> (next: ^Block_Header) { - next = block_next(block) - next.prev_phys_block = block - return -} - -block_mark_as_free :: proc(block: ^Block_Header) { - // Link the block to the next block, first. - next := block_link_next(block) - block_set_prev_free(next) - block_set_free(block) -} - -block_mark_as_used :: proc(block: ^Block_Header) { - next := block_next(block) - block_set_prev_used(next) - block_set_used(block) -} - -@(require_results) -align_up :: proc(x, align: uint) -> (aligned: uint) { - assert(0 == (align & (align - 1)), "must align to a power of two") - return (x + (align - 1)) &~ (align - 1) -} - -@(require_results) -align_down :: proc(x, align: uint) -> (aligned: uint) { - assert(0 == (align & (align - 1)), "must align to a power of two") - return x - (x & (align - 1)) -} - -@(require_results) -align_ptr :: proc(ptr: rawptr, align: uint) -> (aligned: rawptr) { - assert(0 == (align & (align - 1)), "must align to a power of two") - align_mask := uintptr(align) - 1 - _ptr := uintptr(ptr) - _aligned := (_ptr + align_mask) &~ (align_mask) - return rawptr(_aligned) -} - -// Adjust an allocation size to be aligned to word size, and no smaller than internal minimum. -@(require_results) -adjust_request_size :: proc(size, align: uint) -> (adjusted: uint) { - if size == 0 { - return 0 - } - - // aligned size must not exceed `BLOCK_SIZE_MAX`, or we'll go out of bounds on `sl_bitmap`. - if aligned := align_up(size, align); aligned < BLOCK_SIZE_MAX { - adjusted = max(aligned, BLOCK_SIZE_MIN) - } - return -} - -// Adjust an allocation size to be aligned to word size, and no smaller than internal minimum. -@(require_results) -adjust_request_size_with_err :: proc(size, align: uint) -> (adjusted: uint, err: runtime.Allocator_Error) { - if size == 0 { - return 0, nil - } - - // aligned size must not exceed `BLOCK_SIZE_MAX`, or we'll go out of bounds on `sl_bitmap`. - if aligned := align_up(size, align); aligned < BLOCK_SIZE_MAX { - adjusted = min(aligned, BLOCK_SIZE_MAX) - } else { - err = .Out_Of_Memory - } - return -} - -// TLSF utility functions. In most cases these are direct translations of -// the documentation in the research paper. - -@(optimization_mode="favor_size", require_results) -mapping_insert :: proc(size: uint) -> (fl, sl: i32) { - if size < SMALL_BLOCK_SIZE { - // Store small blocks in first list. - sl = i32(size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT) - } else { - fl = fls_uint(size) - sl = i32(size >> (uint(fl) - TLSF_SL_INDEX_COUNT_LOG2)) ~ (1 << TLSF_SL_INDEX_COUNT_LOG2) - fl -= (FL_INDEX_SHIFT - 1) - } - return -} - -@(optimization_mode="favor_size", require_results) -mapping_round :: #force_inline proc(size: uint) -> (rounded: uint) { - rounded = size - if size >= SMALL_BLOCK_SIZE { - round := uint(1 << (uint(fls_uint(size) - TLSF_SL_INDEX_COUNT_LOG2))) - 1 - rounded += round - } - return -} - -// This version rounds up to the next block size (for allocations) -@(optimization_mode="favor_size", require_results) -mapping_search :: proc(size: uint) -> (fl, sl: i32) { - return mapping_insert(mapping_round(size)) -} - -@(require_results) -search_suitable_block :: proc(control: ^Allocator, fli, sli: ^i32) -> (block: ^Block_Header) { - // First, search for a block in the list associated with the given fl/sl index. - fl := fli^; sl := sli^ - - sl_map := control.sl_bitmap[fli^] & (~u32(0) << uint(sl)) - if sl_map == 0 { - // No block exists. Search in the next largest first-level list. - fl_map := control.fl_bitmap & (~u32(0) << uint(fl + 1)) - if fl_map == 0 { - // No free blocks available, memory has been exhausted. - return {} - } - - fl = ffs(fl_map) - fli^ = fl - sl_map = control.sl_bitmap[fl] - } - assert(sl_map != 0, "internal error - second level bitmap is null") - sl = ffs(sl_map) - sli^ = sl - - // Return the first block in the free list. - return control.blocks[fl][sl] -} - -// Remove a free block from the free list. -remove_free_block :: proc(control: ^Allocator, block: ^Block_Header, fl: i32, sl: i32) { - prev := block.prev_free - next := block.next_free - assert(prev != nil, "prev_free can not be nil") - assert(next != nil, "next_free can not be nil") - next.prev_free = prev - prev.next_free = next - - // If this block is the head of the free list, set new head. - if control.blocks[fl][sl] == block { - control.blocks[fl][sl] = next - - // If the new head is nil, clear the bitmap - if next == &control.block_null { - control.sl_bitmap[fl] &~= (u32(1) << uint(sl)) - - // If the second bitmap is now empty, clear the fl bitmap - if control.sl_bitmap[fl] == 0 { - control.fl_bitmap &~= (u32(1) << uint(fl)) - } - } - } -} - -// Insert a free block into the free block list. -insert_free_block :: proc(control: ^Allocator, block: ^Block_Header, fl: i32, sl: i32) { - current := control.blocks[fl][sl] - assert(current != nil, "free lists cannot have a nil entry") - assert(block != nil, "cannot insert a nil entry into the free list") - block.next_free = current - block.prev_free = &control.block_null - current.prev_free = block - - assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE), "block not properly aligned") - - // Insert the new block at the head of the list, and mark the first- and second-level bitmaps appropriately. - control.blocks[fl][sl] = block - control.fl_bitmap |= (u32(1) << uint(fl)) - control.sl_bitmap[fl] |= (u32(1) << uint(sl)) -} - -// Remove a given block from the free list. -block_remove :: proc(control: ^Allocator, block: ^Block_Header) { - fl, sl := mapping_insert(block_size(block)) - remove_free_block(control, block, fl, sl) -} - -// Insert a given block into the free list. -block_insert :: proc(control: ^Allocator, block: ^Block_Header) { - fl, sl := mapping_insert(block_size(block)) - insert_free_block(control, block, fl, sl) -} - -@(require_results) -block_can_split :: proc(block: ^Block_Header, size: uint) -> (can_split: bool) { - return block_size(block) >= size_of(Block_Header) + size -} - -// Split a block into two, the second of which is free. -@(require_results) -block_split :: proc(block: ^Block_Header, size: uint) -> (remaining: ^Block_Header) { - // Calculate the amount of space left in the remaining block. - remaining = offset_to_block(block_to_ptr(block), size - BLOCK_HEADER_OVERHEAD) - - remain_size := block_size(block) - (size + BLOCK_HEADER_OVERHEAD) - - assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE), - "remaining block not aligned properly") - - assert(block_size(block) == remain_size + size + BLOCK_HEADER_OVERHEAD) - block_set_size(remaining, remain_size) - assert(block_size(remaining) >= BLOCK_SIZE_MIN, "block split with invalid size") - - block_set_size(block, size) - block_mark_as_free(remaining) - - return remaining -} - -// Absorb a free block's storage into an adjacent previous free block. -@(require_results) -block_absorb :: proc(prev: ^Block_Header, block: ^Block_Header) -> (absorbed: ^Block_Header) { - assert(!block_is_last(prev), "previous block can't be last") - // Note: Leaves flags untouched. - prev.size += block_size(block) + BLOCK_HEADER_OVERHEAD - _ = block_link_next(prev) - return prev -} - -// Merge a just-freed block with an adjacent previous free block. -@(require_results) -block_merge_prev :: proc(control: ^Allocator, block: ^Block_Header) -> (merged: ^Block_Header) { - merged = block - if (block_is_prev_free(block)) { - prev := block_prev(block) - assert(prev != nil, "prev physical block can't be nil") - assert(block_is_free(prev), "prev block is not free though marked as such") - block_remove(control, prev) - merged = block_absorb(prev, block) - } - return merged -} - -// Merge a just-freed block with an adjacent free block. -@(require_results) -block_merge_next :: proc(control: ^Allocator, block: ^Block_Header) -> (merged: ^Block_Header) { - merged = block - next := block_next(block) - assert(next != nil, "next physical block can't be nil") - - if (block_is_free(next)) { - assert(!block_is_last(block), "previous block can't be last") - block_remove(control, next) - merged = block_absorb(block, next) - } - return merged -} - -// Trim any trailing block space off the end of a free block, return to pool. -block_trim_free :: proc(control: ^Allocator, block: ^Block_Header, size: uint) { - assert(block_is_free(block), "block must be free") - if (block_can_split(block, size)) { - remaining_block := block_split(block, size) - _ = block_link_next(block) - block_set_prev_free(remaining_block) - block_insert(control, remaining_block) - } -} - -// Trim any trailing block space off the end of a used block, return to pool. -block_trim_used :: proc(control: ^Allocator, block: ^Block_Header, size: uint) { - assert(!block_is_free(block), "Block must be used") - if (block_can_split(block, size)) { - // If the next block is free, we must coalesce. - remaining_block := block_split(block, size) - block_set_prev_used(remaining_block) - - remaining_block = block_merge_next(control, remaining_block) - block_insert(control, remaining_block) - } -} - -// Trim leading block space, return to pool. -@(require_results) -block_trim_free_leading :: proc(control: ^Allocator, block: ^Block_Header, size: uint) -> (remaining: ^Block_Header) { - remaining = block - if block_can_split(block, size) { - // We want the 2nd block. - remaining = block_split(block, size - BLOCK_HEADER_OVERHEAD) - block_set_prev_free(remaining) - - _ = block_link_next(block) - block_insert(control, block) - } - return remaining -} - -@(require_results) -block_locate_free :: proc(control: ^Allocator, size: uint) -> (block: ^Block_Header) { - fl, sl: i32 - if size != 0 { - fl, sl = mapping_search(size) - - /* - `mapping_search` can futz with the size, so for excessively large sizes it can sometimes wind up - with indices that are off the end of the block array. So, we protect against that here, - since this is the only call site of `mapping_search`. Note that we don't need to check `sl`, - as it comes from a modulo operation that guarantees it's always in range. - */ - if fl < FL_INDEX_COUNT { - block = search_suitable_block(control, &fl, &sl) - } - } - - if block != nil { - assert(block_size(block) >= size) - remove_free_block(control, block, fl, sl) - } - return block -} - -@(require_results) -block_prepare_used :: proc(control: ^Allocator, block: ^Block_Header, size: uint) -> (res: []byte, err: runtime.Allocator_Error) { - if block != nil { - assert(size != 0, "Size must be non-zero") - block_trim_free(control, block, size) - block_mark_as_used(block) - res = ([^]byte)(block_to_ptr(block))[:size] - } - return -} - // Clear control structure and point all empty lists at the null block -clear :: proc(control: ^Allocator) { +@(private) +free_all :: proc(control: ^Allocator) -> (err: Error) { + // Clear internal structures control.block_null.next_free = &control.block_null control.block_null.prev_free = &control.block_null - control.fl_bitmap = 0 for i in 0.. (err: Error) { assert(uintptr(raw_data(pool)) % ALIGN_SIZE == 0, "Added memory must be aligned") @@ -574,9 +154,11 @@ pool_add :: proc(control: ^Allocator, pool: []u8) -> (err: Error) { block_set_size(next, 0) block_set_used(next) block_set_prev_free(next) + return } +@(private) pool_remove :: proc(control: ^Allocator, pool: []u8) { block := offset_to_block_backwards(raw_data(pool), BLOCK_HEADER_OVERHEAD) @@ -588,7 +170,7 @@ pool_remove :: proc(control: ^Allocator, pool: []u8) { remove_free_block(control, block, fl, sl) } -@(require_results) +@(private, require_results) alloc_bytes_non_zeroed :: proc(control: ^Allocator, size: uint, align: uint) -> (res: []byte, err: runtime.Allocator_Error) { assert(control != nil) adjust := adjust_request_size(size, ALIGN_SIZE) @@ -601,9 +183,54 @@ alloc_bytes_non_zeroed :: proc(control: ^Allocator, size: uint, align: uint) -> return nil, .Out_Of_Memory } - block := block_locate_free(control, aligned_size) + block := block_locate_free(control, aligned_size) if block == nil { - return nil, .Out_Of_Memory + // OOM: Couldn't find block of `aligned_size` bytes. + if control.new_pool_size > 0 && control.pool.allocator.procedure != nil { + // TLSF is configured to grow. + + /* + This implementation doesn't allow for out-of-band allocations to be passed through, as it's not designed to + track those. Nor is it able to signal those allocations then need to be freed on the backing allocator, + as opposed to regular allocations handled for you when you `destroy` the TLSF instance. + + So if we're asked for more than we're configured to grow by, we can fail with an OOM error early, without adding a new pool. + */ + if aligned_size > control.new_pool_size { + return nil, .Out_Of_Memory + } + + // Trying to allocate a new pool of `control.new_pool_size` bytes. + new_pool_buf := runtime.make_aligned([]byte, control.new_pool_size, ALIGN_SIZE, control.pool.allocator) or_return + + // Add new pool to control structure + if pool_add_err := pool_add(control, new_pool_buf); pool_add_err != .None { + delete(new_pool_buf, control.pool.allocator) + return nil, .Out_Of_Memory + } + + // Allocate a new link in the `control.pool` tracking structure. + new_pool := new_clone(Pool{ + data = new_pool_buf, + allocator = control.pool.allocator, + next = nil, + }, control.pool.allocator) or_return + + p := &control.pool + for p.next != nil { + p = p.next + } + p.next = new_pool + + // Try again to find free block + block = block_locate_free(control, aligned_size) + if block == nil { + return nil, .Out_Of_Memory + } + } else { + // TLSF is non-growing. We're done. + return nil, .Out_Of_Memory + } } ptr := block_to_ptr(block) aligned := align_ptr(ptr, align) @@ -627,7 +254,7 @@ alloc_bytes_non_zeroed :: proc(control: ^Allocator, size: uint, align: uint) -> return block_prepare_used(control, block, adjust) } -@(require_results) +@(private, require_results) alloc_bytes :: proc(control: ^Allocator, size: uint, align: uint) -> (res: []byte, err: runtime.Allocator_Error) { res, err = alloc_bytes_non_zeroed(control, size, align) if err == nil { @@ -653,7 +280,7 @@ free_with_size :: proc(control: ^Allocator, ptr: rawptr, size: uint) { } -@(require_results) +@(private, require_results) resize :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: uint, alignment: uint) -> (res: []byte, err: runtime.Allocator_Error) { assert(control != nil) if ptr != nil && new_size == 0 { @@ -697,7 +324,7 @@ resize :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: uint, align return } -@(require_results) +@(private, require_results) resize_non_zeroed :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: uint, alignment: uint) -> (res: []byte, err: runtime.Allocator_Error) { assert(control != nil) if ptr != nil && new_size == 0 { @@ -736,3 +363,424 @@ resize_non_zeroed :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: res = ([^]byte)(ptr)[:new_size] return } + +/* + TLSF achieves O(1) cost for `alloc` and `free` operations by limiting + the search for a free block to a free list of guaranteed size + adequate to fulfill the request, combined with efficient free list + queries using bitmasks and architecture-specific bit-manipulation + routines. + + NOTE: TLSF spec relies on ffs/fls returning a value in the range 0..31. +*/ + +@(private, require_results) +block_size :: proc "contextless" (block: ^Block_Header) -> (size: uint) { + return block.size &~ (BLOCK_HEADER_FREE | BLOCK_HEADER_PREV_FREE) +} + +@(private) +block_set_size :: proc "contextless" (block: ^Block_Header, size: uint) { + old_size := block.size + block.size = size | (old_size & (BLOCK_HEADER_FREE | BLOCK_HEADER_PREV_FREE)) +} + +@(private, require_results) +block_is_last :: proc "contextless" (block: ^Block_Header) -> (is_last: bool) { + return block_size(block) == 0 +} + +@(private, require_results) +block_is_free :: proc "contextless" (block: ^Block_Header) -> (is_free: bool) { + return (block.size & BLOCK_HEADER_FREE) == BLOCK_HEADER_FREE +} + +@(private) +block_set_free :: proc "contextless" (block: ^Block_Header) { + block.size |= BLOCK_HEADER_FREE +} + +@(private) +block_set_used :: proc "contextless" (block: ^Block_Header) { + block.size &~= BLOCK_HEADER_FREE +} + +@(private, require_results) +block_is_prev_free :: proc "contextless" (block: ^Block_Header) -> (is_prev_free: bool) { + return (block.size & BLOCK_HEADER_PREV_FREE) == BLOCK_HEADER_PREV_FREE +} + +@(private) +block_set_prev_free :: proc "contextless" (block: ^Block_Header) { + block.size |= BLOCK_HEADER_PREV_FREE +} + +@(private) +block_set_prev_used :: proc "contextless" (block: ^Block_Header) { + block.size &~= BLOCK_HEADER_PREV_FREE +} + +@(private, require_results) +block_from_ptr :: proc(ptr: rawptr) -> (block_ptr: ^Block_Header) { + return (^Block_Header)(uintptr(ptr) - BLOCK_START_OFFSET) +} + +@(private, require_results) +block_to_ptr :: proc(block: ^Block_Header) -> (ptr: rawptr) { + return rawptr(uintptr(block) + BLOCK_START_OFFSET) +} + +// Return location of next block after block of given size. +@(private, require_results) +offset_to_block :: proc(ptr: rawptr, size: uint) -> (block: ^Block_Header) { + return (^Block_Header)(uintptr(ptr) + uintptr(size)) +} + +@(private, require_results) +offset_to_block_backwards :: proc(ptr: rawptr, size: uint) -> (block: ^Block_Header) { + return (^Block_Header)(uintptr(ptr) - uintptr(size)) +} + +// Return location of previous block. +@(private, require_results) +block_prev :: proc(block: ^Block_Header) -> (prev: ^Block_Header) { + assert(block_is_prev_free(block), "previous block must be free") + return block.prev_phys_block +} + +// Return location of next existing block. +@(private, require_results) +block_next :: proc(block: ^Block_Header) -> (next: ^Block_Header) { + return offset_to_block(block_to_ptr(block), block_size(block) - BLOCK_HEADER_OVERHEAD) +} + +// Link a new block with its physical neighbor, return the neighbor. +@(private, require_results) +block_link_next :: proc(block: ^Block_Header) -> (next: ^Block_Header) { + next = block_next(block) + next.prev_phys_block = block + return +} + +@(private) +block_mark_as_free :: proc(block: ^Block_Header) { + // Link the block to the next block, first. + next := block_link_next(block) + block_set_prev_free(next) + block_set_free(block) +} + +@(private) +block_mark_as_used :: proc(block: ^Block_Header) { + next := block_next(block) + block_set_prev_used(next) + block_set_used(block) +} + +@(private, require_results) +align_up :: proc(x, align: uint) -> (aligned: uint) { + assert(0 == (align & (align - 1)), "must align to a power of two") + return (x + (align - 1)) &~ (align - 1) +} + +@(private, require_results) +align_down :: proc(x, align: uint) -> (aligned: uint) { + assert(0 == (align & (align - 1)), "must align to a power of two") + return x - (x & (align - 1)) +} + +@(private, require_results) +align_ptr :: proc(ptr: rawptr, align: uint) -> (aligned: rawptr) { + assert(0 == (align & (align - 1)), "must align to a power of two") + align_mask := uintptr(align) - 1 + _ptr := uintptr(ptr) + _aligned := (_ptr + align_mask) &~ (align_mask) + return rawptr(_aligned) +} + +// Adjust an allocation size to be aligned to word size, and no smaller than internal minimum. +@(private, require_results) +adjust_request_size :: proc(size, align: uint) -> (adjusted: uint) { + if size == 0 { + return 0 + } + + // aligned size must not exceed `BLOCK_SIZE_MAX`, or we'll go out of bounds on `sl_bitmap`. + if aligned := align_up(size, align); aligned < BLOCK_SIZE_MAX { + adjusted = max(aligned, BLOCK_SIZE_MIN) + } + return +} + +// Adjust an allocation size to be aligned to word size, and no smaller than internal minimum. +@(private, require_results) +adjust_request_size_with_err :: proc(size, align: uint) -> (adjusted: uint, err: runtime.Allocator_Error) { + if size == 0 { + return 0, nil + } + + // aligned size must not exceed `BLOCK_SIZE_MAX`, or we'll go out of bounds on `sl_bitmap`. + if aligned := align_up(size, align); aligned < BLOCK_SIZE_MAX { + adjusted = min(aligned, BLOCK_SIZE_MAX) + } else { + err = .Out_Of_Memory + } + return +} + +// TLSF utility functions. In most cases these are direct translations of +// the documentation in the research paper. + +@(optimization_mode="favor_size", private, require_results) +mapping_insert :: proc(size: uint) -> (fl, sl: i32) { + if size < SMALL_BLOCK_SIZE { + // Store small blocks in first list. + sl = i32(size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT) + } else { + fl = fls_uint(size) + sl = i32(size >> (uint(fl) - TLSF_SL_INDEX_COUNT_LOG2)) ~ (1 << TLSF_SL_INDEX_COUNT_LOG2) + fl -= (FL_INDEX_SHIFT - 1) + } + return +} + +@(optimization_mode="favor_size", private, require_results) +mapping_round :: #force_inline proc(size: uint) -> (rounded: uint) { + rounded = size + if size >= SMALL_BLOCK_SIZE { + round := uint(1 << (uint(fls_uint(size) - TLSF_SL_INDEX_COUNT_LOG2))) - 1 + rounded += round + } + return +} + +// This version rounds up to the next block size (for allocations) +@(optimization_mode="favor_size", private, require_results) +mapping_search :: proc(size: uint) -> (fl, sl: i32) { + return mapping_insert(mapping_round(size)) +} + +@(private, require_results) +search_suitable_block :: proc(control: ^Allocator, fli, sli: ^i32) -> (block: ^Block_Header) { + // First, search for a block in the list associated with the given fl/sl index. + fl := fli^; sl := sli^ + + sl_map := control.sl_bitmap[fli^] & (~u32(0) << uint(sl)) + if sl_map == 0 { + // No block exists. Search in the next largest first-level list. + fl_map := control.fl_bitmap & (~u32(0) << uint(fl + 1)) + if fl_map == 0 { + // No free blocks available, memory has been exhausted. + return {} + } + + fl = ffs(fl_map) + fli^ = fl + sl_map = control.sl_bitmap[fl] + } + assert(sl_map != 0, "internal error - second level bitmap is null") + sl = ffs(sl_map) + sli^ = sl + + // Return the first block in the free list. + return control.blocks[fl][sl] +} + +// Remove a free block from the free list. +@(private) +remove_free_block :: proc(control: ^Allocator, block: ^Block_Header, fl: i32, sl: i32) { + prev := block.prev_free + next := block.next_free + assert(prev != nil, "prev_free can not be nil") + assert(next != nil, "next_free can not be nil") + next.prev_free = prev + prev.next_free = next + + // If this block is the head of the free list, set new head. + if control.blocks[fl][sl] == block { + control.blocks[fl][sl] = next + + // If the new head is nil, clear the bitmap + if next == &control.block_null { + control.sl_bitmap[fl] &~= (u32(1) << uint(sl)) + + // If the second bitmap is now empty, clear the fl bitmap + if control.sl_bitmap[fl] == 0 { + control.fl_bitmap &~= (u32(1) << uint(fl)) + } + } + } +} + +// Insert a free block into the free block list. +@(private) +insert_free_block :: proc(control: ^Allocator, block: ^Block_Header, fl: i32, sl: i32) { + current := control.blocks[fl][sl] + assert(current != nil, "free lists cannot have a nil entry") + assert(block != nil, "cannot insert a nil entry into the free list") + block.next_free = current + block.prev_free = &control.block_null + current.prev_free = block + + assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE), "block not properly aligned") + + // Insert the new block at the head of the list, and mark the first- and second-level bitmaps appropriately. + control.blocks[fl][sl] = block + control.fl_bitmap |= (u32(1) << uint(fl)) + control.sl_bitmap[fl] |= (u32(1) << uint(sl)) +} + +// Remove a given block from the free list. +@(private) +block_remove :: proc(control: ^Allocator, block: ^Block_Header) { + fl, sl := mapping_insert(block_size(block)) + remove_free_block(control, block, fl, sl) +} + +// Insert a given block into the free list. +@(private) +block_insert :: proc(control: ^Allocator, block: ^Block_Header) { + fl, sl := mapping_insert(block_size(block)) + insert_free_block(control, block, fl, sl) +} + +@(private, require_results) +block_can_split :: proc(block: ^Block_Header, size: uint) -> (can_split: bool) { + return block_size(block) >= size_of(Block_Header) + size +} + +// Split a block into two, the second of which is free. +@(private, require_results) +block_split :: proc(block: ^Block_Header, size: uint) -> (remaining: ^Block_Header) { + // Calculate the amount of space left in the remaining block. + remaining = offset_to_block(block_to_ptr(block), size - BLOCK_HEADER_OVERHEAD) + + remain_size := block_size(block) - (size + BLOCK_HEADER_OVERHEAD) + + assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE), + "remaining block not aligned properly") + + assert(block_size(block) == remain_size + size + BLOCK_HEADER_OVERHEAD) + block_set_size(remaining, remain_size) + assert(block_size(remaining) >= BLOCK_SIZE_MIN, "block split with invalid size") + + block_set_size(block, size) + block_mark_as_free(remaining) + + return remaining +} + +// Absorb a free block's storage into an adjacent previous free block. +@(private, require_results) +block_absorb :: proc(prev: ^Block_Header, block: ^Block_Header) -> (absorbed: ^Block_Header) { + assert(!block_is_last(prev), "previous block can't be last") + // Note: Leaves flags untouched. + prev.size += block_size(block) + BLOCK_HEADER_OVERHEAD + _ = block_link_next(prev) + return prev +} + +// Merge a just-freed block with an adjacent previous free block. +@(private, require_results) +block_merge_prev :: proc(control: ^Allocator, block: ^Block_Header) -> (merged: ^Block_Header) { + merged = block + if (block_is_prev_free(block)) { + prev := block_prev(block) + assert(prev != nil, "prev physical block can't be nil") + assert(block_is_free(prev), "prev block is not free though marked as such") + block_remove(control, prev) + merged = block_absorb(prev, block) + } + return merged +} + +// Merge a just-freed block with an adjacent free block. +@(private, require_results) +block_merge_next :: proc(control: ^Allocator, block: ^Block_Header) -> (merged: ^Block_Header) { + merged = block + next := block_next(block) + assert(next != nil, "next physical block can't be nil") + + if (block_is_free(next)) { + assert(!block_is_last(block), "previous block can't be last") + block_remove(control, next) + merged = block_absorb(block, next) + } + return merged +} + +// Trim any trailing block space off the end of a free block, return to pool. +@(private) +block_trim_free :: proc(control: ^Allocator, block: ^Block_Header, size: uint) { + assert(block_is_free(block), "block must be free") + if (block_can_split(block, size)) { + remaining_block := block_split(block, size) + _ = block_link_next(block) + block_set_prev_free(remaining_block) + block_insert(control, remaining_block) + } +} + +// Trim any trailing block space off the end of a used block, return to pool. +@(private) +block_trim_used :: proc(control: ^Allocator, block: ^Block_Header, size: uint) { + assert(!block_is_free(block), "Block must be used") + if (block_can_split(block, size)) { + // If the next block is free, we must coalesce. + remaining_block := block_split(block, size) + block_set_prev_used(remaining_block) + + remaining_block = block_merge_next(control, remaining_block) + block_insert(control, remaining_block) + } +} + +// Trim leading block space, return to pool. +@(private, require_results) +block_trim_free_leading :: proc(control: ^Allocator, block: ^Block_Header, size: uint) -> (remaining: ^Block_Header) { + remaining = block + if block_can_split(block, size) { + // We want the 2nd block. + remaining = block_split(block, size - BLOCK_HEADER_OVERHEAD) + block_set_prev_free(remaining) + + _ = block_link_next(block) + block_insert(control, block) + } + return remaining +} + +@(private, require_results) +block_locate_free :: proc(control: ^Allocator, size: uint) -> (block: ^Block_Header) { + fl, sl: i32 + if size != 0 { + fl, sl = mapping_search(size) + + /* + `mapping_search` can futz with the size, so for excessively large sizes it can sometimes wind up + with indices that are off the end of the block array. So, we protect against that here, + since this is the only call site of `mapping_search`. Note that we don't need to check `sl`, + as it comes from a modulo operation that guarantees it's always in range. + */ + if fl < FL_INDEX_COUNT { + block = search_suitable_block(control, &fl, &sl) + } + } + + if block != nil { + assert(block_size(block) >= size) + remove_free_block(control, block, fl, sl) + } + return block +} + +@(private, require_results) +block_prepare_used :: proc(control: ^Allocator, block: ^Block_Header, size: uint) -> (res: []byte, err: runtime.Allocator_Error) { + if block != nil { + assert(size != 0, "Size must be non-zero") + block_trim_free(control, block, size) + block_mark_as_used(block) + res = ([^]byte)(block_to_ptr(block))[:size] + } + return +} \ No newline at end of file diff --git a/core/net/errors_linux.odin b/core/net/errors_linux.odin index da9811d71..237579f28 100644 --- a/core/net/errors_linux.odin +++ b/core/net/errors_linux.odin @@ -159,7 +159,7 @@ _tcp_recv_error :: proc(errno: linux.Errno) -> TCP_Recv_Error { return .Invalid_Argument case .ENOTCONN: return .Not_Connected - case .ECONNREFUSED: + case .ECONNREFUSED, .ECONNRESET: return .Connection_Closed case .ETIMEDOUT: return .Timeout @@ -179,7 +179,7 @@ _udp_recv_error :: proc(errno: linux.Errno) -> UDP_Recv_Error { #partial switch errno { case .EBADF, .ENOTSOCK, .EFAULT: return .Invalid_Argument - case .ECONNREFUSED, .ENOTCONN: + case .ECONNREFUSED, .ENOTCONN, .ECONNRESET: return .Connection_Refused case .ETIMEDOUT: return .Timeout diff --git a/tests/core/mem/test_core_mem.odin b/tests/core/mem/test_core_mem.odin index 4f2095fca..bd072b4e9 100644 --- a/tests/core/mem/test_core_mem.odin +++ b/tests/core/mem/test_core_mem.odin @@ -1,8 +1,10 @@ package test_core_mem +import "core:mem" import "core:mem/tlsf" import "core:mem/virtual" import "core:testing" +import "core:slice" @test test_tlsf_bitscan :: proc(t: ^testing.T) { @@ -54,3 +56,140 @@ test_align_bumping_block_limit :: proc(t: ^testing.T) { testing.expect_value(t, err, nil) testing.expect(t, len(data) == 896) } + +@(test) +tlsf_test_overlap_and_zero :: proc(t: ^testing.T) { + default_allocator := context.allocator + alloc: tlsf.Allocator + defer tlsf.destroy(&alloc) + + NUM_ALLOCATIONS :: 1_000 + BACKING_SIZE :: NUM_ALLOCATIONS * (1_000 + size_of(uintptr)) + + if err := tlsf.init_from_allocator(&alloc, default_allocator, BACKING_SIZE); err != .None { + testing.fail_now(t, "TLSF init error") + } + context.allocator = tlsf.allocator(&alloc) + + allocations := make([dynamic][]byte, 0, NUM_ALLOCATIONS, default_allocator) + defer delete(allocations) + + err: mem.Allocator_Error + s: []byte + + for size := 1; err == .None && size <= NUM_ALLOCATIONS; size += 1 { + s, err = make([]byte, size) + append(&allocations, s) + } + + slice.sort_by(allocations[:], proc(a, b: []byte) -> bool { + return uintptr(raw_data(a)) < uintptr(raw_data((b))) + }) + + for i in 0.. bool { + return uintptr(raw_data(a)) < uintptr(raw_data((b))) + }) + + for i in 0..= 10) + + free_all(tlsf.allocator(&alloc)) + + for { + s := make([]byte, ALLOCATION_SIZE) or_break + append(&allocations[1], s) + } + testing.expect(t, len(allocations[1]) >= 10) + + for i in 0..= b_end && b_end >= a_start { + testing.fail_now(t, "Allocations overlapped") + } +}