mirror of
https://github.com/odin-lang/Odin.git
synced 2026-02-15 07:43:13 +00:00
os2/heap_linux point to runtime._heap_allocator_proc
This commit is contained in:
jason
@@ -76,7 +76,7 @@ _set_env :: proc(key, v_new: string) -> bool {
|
||||
// wasn't in the environment in the first place.
|
||||
k_addr, v_addr := _kv_addr_from_val(v_curr, key)
|
||||
if len(v_new) > len(v_curr) {
|
||||
k_addr = ([^]u8)(heap_resize(k_addr, kv_size))
|
||||
k_addr = ([^]u8)(runtime.heap_resize(k_addr, kv_size))
|
||||
if k_addr == nil {
|
||||
return false
|
||||
}
|
||||
@@ -90,7 +90,7 @@ _set_env :: proc(key, v_new: string) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
k_addr := ([^]u8)(heap_alloc(kv_size))
|
||||
k_addr := ([^]u8)(runtime.heap_alloc(kv_size))
|
||||
if k_addr == nil {
|
||||
return false
|
||||
}
|
||||
@@ -129,7 +129,7 @@ _unset_env :: proc(key: string) -> bool {
|
||||
// if we got this far, the envrionment variable
|
||||
// existed AND was allocated by us.
|
||||
k_addr, _ := _kv_addr_from_val(v, key)
|
||||
heap_free(k_addr)
|
||||
runtime.heap_free(k_addr)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -139,7 +139,7 @@ _clear_env :: proc() {
|
||||
|
||||
for kv in _env {
|
||||
if !_is_in_org_env(kv) {
|
||||
heap_free(raw_data(kv))
|
||||
runtime.heap_free(raw_data(kv))
|
||||
}
|
||||
}
|
||||
clear(&_env)
|
||||
@@ -193,7 +193,7 @@ _build_env :: proc() {
|
||||
return
|
||||
}
|
||||
|
||||
_env = make(type_of(_env), heap_allocator())
|
||||
_env = make(type_of(_env), runtime.heap_allocator())
|
||||
cstring_env := _get_original_env()
|
||||
_org_env_begin = uintptr(rawptr(cstring_env[0]))
|
||||
for i := 0; cstring_env[i] != nil; i += 1 {
|
||||
|
||||
@@ -1,726 +1,6 @@
|
||||
#+private
|
||||
package os2
|
||||
|
||||
import "core:sys/linux"
|
||||
import "core:sync"
|
||||
import "core:mem"
|
||||
|
||||
// NOTEs
|
||||
//
|
||||
// All allocations below DIRECT_MMAP_THRESHOLD exist inside of memory "Regions." A region
|
||||
// consists of a Region_Header and the memory that will be divided into allocations to
|
||||
// send to the user. The memory is an array of "Allocation_Headers" which are 8 bytes.
|
||||
// Allocation_Headers are used to navigate the memory in the region. The "next" member of
|
||||
// the Allocation_Header points to the next header, and the space between the headers
|
||||
// can be used to send to the user. This space between is referred to as "blocks" in the
|
||||
// code. The indexes in the header refer to these blocks instead of bytes. This allows us
|
||||
// to index all the memory in the region with a u16.
|
||||
//
|
||||
// When an allocation request is made, it will use the first free block that can contain
|
||||
// the entire block. If there is an excess number of blocks (as specified by the constant
|
||||
// BLOCK_SEGMENT_THRESHOLD), this extra space will be segmented and left in the free_list.
|
||||
//
|
||||
// To keep the implementation simple, there can never exist 2 free blocks adjacent to each
|
||||
// other. Any freeing will result in attempting to merge the blocks before and after the
|
||||
// newly free'd blocks.
|
||||
//
|
||||
// Any request for size above the DIRECT_MMAP_THRESHOLD will result in the allocation
|
||||
// getting its own individual mmap. Individual mmaps will still get an Allocation_Header
|
||||
// that contains the size with the last bit set to 1 to indicate it is indeed a direct
|
||||
// mmap allocation.
|
||||
|
||||
// Why not brk?
|
||||
// glibc's malloc utilizes a mix of the brk and mmap system calls. This implementation
|
||||
// does *not* utilize the brk system call to avoid possible conflicts with foreign C
|
||||
// code. Just because we aren't directly using libc, there is nothing stopping the user
|
||||
// from doing it.
|
||||
|
||||
// What's with all the #no_bounds_check?
|
||||
// When memory is returned from mmap, it technically doesn't get written ... well ... anywhere
|
||||
// until that region is written to by *you*. So, when a new region is created, we call mmap
|
||||
// to get a pointer to some memory, and we claim that memory is a ^Region. Therefor, the
|
||||
// region itself is never formally initialized by the compiler as this would result in writing
|
||||
// zeros to memory that we can already assume are 0. This would also have the effect of
|
||||
// actually commiting this data to memory whether it gets used or not.
|
||||
|
||||
|
||||
//
|
||||
// Some variables to play with
|
||||
//
|
||||
|
||||
// Minimum blocks used for any one allocation
|
||||
MINIMUM_BLOCK_COUNT :: 2
|
||||
|
||||
// Number of extra blocks beyond the requested amount where we would segment.
|
||||
// E.g. (blocks) |H0123456| 7 available
|
||||
// |H01H0123| Ask for 2, now 4 available
|
||||
BLOCK_SEGMENT_THRESHOLD :: 4
|
||||
|
||||
// Anything above this threshold will get its own memory map. Since regions
|
||||
// are indexed by 16 bit integers, this value should not surpass max(u16) * 6
|
||||
DIRECT_MMAP_THRESHOLD_USER :: int(max(u16))
|
||||
|
||||
// The point at which we convert direct mmap to region. This should be a decent
|
||||
// amount less than DIRECT_MMAP_THRESHOLD to avoid jumping in and out of regions.
|
||||
MMAP_TO_REGION_SHRINK_THRESHOLD :: DIRECT_MMAP_THRESHOLD - PAGE_SIZE * 4
|
||||
|
||||
// free_list is dynamic and is initialized in the begining of the region memory
|
||||
// when the region is initialized. Once resized, it can be moved anywhere.
|
||||
FREE_LIST_DEFAULT_CAP :: 32
|
||||
|
||||
|
||||
//
|
||||
// Other constants that should not be touched
|
||||
//
|
||||
|
||||
// This universally seems to be 4096 outside of uncommon archs.
|
||||
PAGE_SIZE :: 4096
|
||||
|
||||
// just rounding up to nearest PAGE_SIZE
|
||||
DIRECT_MMAP_THRESHOLD :: (DIRECT_MMAP_THRESHOLD_USER-1) + PAGE_SIZE - (DIRECT_MMAP_THRESHOLD_USER-1) % PAGE_SIZE
|
||||
|
||||
// Regions must be big enough to hold DIRECT_MMAP_THRESHOLD - 1 as well
|
||||
// as end right on a page boundary as to not waste space.
|
||||
SIZE_OF_REGION :: DIRECT_MMAP_THRESHOLD + 4 * int(PAGE_SIZE)
|
||||
|
||||
// size of user memory blocks
|
||||
BLOCK_SIZE :: size_of(Allocation_Header)
|
||||
|
||||
// number of allocation sections (call them blocks) of the region used for allocations
|
||||
BLOCKS_PER_REGION :: u16((SIZE_OF_REGION - size_of(Region_Header)) / BLOCK_SIZE)
|
||||
|
||||
// minimum amount of space that can used by any individual allocation (includes header)
|
||||
MINIMUM_ALLOCATION :: (MINIMUM_BLOCK_COUNT * BLOCK_SIZE) + BLOCK_SIZE
|
||||
|
||||
// This is used as a boolean value for Region_Header.local_addr.
|
||||
CURRENTLY_ACTIVE :: (^^Region)(~uintptr(0))
|
||||
|
||||
FREE_LIST_ENTRIES_PER_BLOCK :: BLOCK_SIZE / size_of(u16)
|
||||
|
||||
MMAP_FLAGS : linux.Map_Flags : {.ANONYMOUS, .PRIVATE}
|
||||
MMAP_PROT : linux.Mem_Protection : {.READ, .WRITE}
|
||||
|
||||
@thread_local _local_region: ^Region
|
||||
global_regions: ^Region
|
||||
|
||||
|
||||
// There is no way of correctly setting the last bit of free_idx or
|
||||
// the last bit of requested, so we can safely use it as a flag to
|
||||
// determine if we are interacting with a direct mmap.
|
||||
REQUESTED_MASK :: 0x7FFFFFFFFFFFFFFF
|
||||
IS_DIRECT_MMAP :: 0x8000000000000000
|
||||
|
||||
// Special free_idx value that does not index the free_list.
|
||||
NOT_FREE :: 0x7FFF
|
||||
Allocation_Header :: struct #raw_union {
|
||||
using _: struct {
|
||||
// Block indicies
|
||||
idx: u16,
|
||||
prev: u16,
|
||||
next: u16,
|
||||
free_idx: u16,
|
||||
},
|
||||
requested: u64,
|
||||
}
|
||||
|
||||
Region_Header :: struct #align(16) {
|
||||
next_region: ^Region, // points to next region in global_heap (linked list)
|
||||
local_addr: ^^Region, // tracks region ownership via address of _local_region
|
||||
reset_addr: ^^Region, // tracks old local addr for reset
|
||||
free_list: []u16,
|
||||
free_list_len: u16,
|
||||
free_blocks: u16, // number of free blocks in region (includes headers)
|
||||
last_used: u16, // farthest back block that has been used (need zeroing?)
|
||||
_reserved: u16,
|
||||
}
|
||||
|
||||
Region :: struct {
|
||||
hdr: Region_Header,
|
||||
memory: [BLOCKS_PER_REGION]Allocation_Header,
|
||||
}
|
||||
|
||||
_heap_allocator_proc :: proc(allocator_data: rawptr, mode: mem.Allocator_Mode,
|
||||
size, alignment: int,
|
||||
old_memory: rawptr, old_size: int, loc := #caller_location) -> ([]byte, mem.Allocator_Error) {
|
||||
//
|
||||
// NOTE(tetra, 2020-01-14): The heap doesn't respect alignment.
|
||||
// Instead, we overallocate by `alignment + size_of(rawptr) - 1`, and insert
|
||||
// padding. We also store the original pointer returned by heap_alloc right before
|
||||
// the pointer we return to the user.
|
||||
//
|
||||
|
||||
aligned_alloc :: proc(size, alignment: int, old_ptr: rawptr = nil) -> ([]byte, mem.Allocator_Error) {
|
||||
a := max(alignment, align_of(rawptr))
|
||||
space := size + a - 1
|
||||
|
||||
allocated_mem: rawptr
|
||||
if old_ptr != nil {
|
||||
original_old_ptr := mem.ptr_offset((^rawptr)(old_ptr), -1)^
|
||||
allocated_mem = heap_resize(original_old_ptr, space+size_of(rawptr))
|
||||
} else {
|
||||
allocated_mem = heap_alloc(space+size_of(rawptr))
|
||||
}
|
||||
aligned_mem := rawptr(mem.ptr_offset((^u8)(allocated_mem), size_of(rawptr)))
|
||||
|
||||
ptr := uintptr(aligned_mem)
|
||||
aligned_ptr := (ptr - 1 + uintptr(a)) & -uintptr(a)
|
||||
diff := int(aligned_ptr - ptr)
|
||||
if (size + diff) > space || allocated_mem == nil {
|
||||
return nil, .Out_Of_Memory
|
||||
}
|
||||
|
||||
aligned_mem = rawptr(aligned_ptr)
|
||||
mem.ptr_offset((^rawptr)(aligned_mem), -1)^ = allocated_mem
|
||||
|
||||
return mem.byte_slice(aligned_mem, size), nil
|
||||
}
|
||||
|
||||
aligned_free :: proc(p: rawptr) {
|
||||
if p != nil {
|
||||
heap_free(mem.ptr_offset((^rawptr)(p), -1)^)
|
||||
}
|
||||
}
|
||||
|
||||
aligned_resize :: proc(p: rawptr, old_size: int, new_size: int, new_alignment: int) -> (new_memory: []byte, err: mem.Allocator_Error) {
|
||||
if p == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return aligned_alloc(new_size, new_alignment, p)
|
||||
}
|
||||
|
||||
switch mode {
|
||||
case .Alloc, .Alloc_Non_Zeroed:
|
||||
return aligned_alloc(size, alignment)
|
||||
|
||||
case .Free:
|
||||
aligned_free(old_memory)
|
||||
|
||||
case .Free_All:
|
||||
return nil, .Mode_Not_Implemented
|
||||
|
||||
case .Resize, .Resize_Non_Zeroed:
|
||||
if old_memory == nil {
|
||||
return aligned_alloc(size, alignment)
|
||||
}
|
||||
return aligned_resize(old_memory, old_size, size, alignment)
|
||||
|
||||
case .Query_Features:
|
||||
set := (^mem.Allocator_Mode_Set)(old_memory)
|
||||
if set != nil {
|
||||
set^ = {.Alloc, .Free, .Resize, .Query_Features}
|
||||
}
|
||||
return nil, nil
|
||||
|
||||
case .Query_Info:
|
||||
return nil, .Mode_Not_Implemented
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
heap_alloc :: proc(size: int) -> rawptr {
|
||||
if size >= DIRECT_MMAP_THRESHOLD {
|
||||
return _direct_mmap_alloc(size)
|
||||
}
|
||||
|
||||
// atomically check if the local region has been stolen
|
||||
if _local_region != nil {
|
||||
res := sync.atomic_compare_exchange_strong_explicit(
|
||||
&_local_region.hdr.local_addr,
|
||||
&_local_region,
|
||||
CURRENTLY_ACTIVE,
|
||||
.Acquire,
|
||||
.Relaxed,
|
||||
)
|
||||
if res != &_local_region {
|
||||
// At this point, the region has been stolen and res contains the unexpected value
|
||||
expected := res
|
||||
if res != CURRENTLY_ACTIVE {
|
||||
expected = res
|
||||
res = sync.atomic_compare_exchange_strong_explicit(
|
||||
&_local_region.hdr.local_addr,
|
||||
expected,
|
||||
CURRENTLY_ACTIVE,
|
||||
.Acquire,
|
||||
.Relaxed,
|
||||
)
|
||||
}
|
||||
if res != expected {
|
||||
_local_region = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size := size
|
||||
size = _round_up_to_nearest(size, BLOCK_SIZE)
|
||||
blocks_needed := u16(max(MINIMUM_BLOCK_COUNT, size / BLOCK_SIZE))
|
||||
|
||||
// retrieve a region if new thread or stolen
|
||||
if _local_region == nil {
|
||||
_local_region, _ = _region_retrieve_with_space(blocks_needed)
|
||||
if _local_region == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
defer sync.atomic_store_explicit(&_local_region.hdr.local_addr, &_local_region, .Release)
|
||||
|
||||
// At this point we have a usable region. Let's find the user some memory
|
||||
idx: u16
|
||||
local_region_idx := _region_get_local_idx()
|
||||
back_idx := -1
|
||||
infinite: for {
|
||||
for i := 0; i < int(_local_region.hdr.free_list_len); i += 1 {
|
||||
idx = _local_region.hdr.free_list[i]
|
||||
#no_bounds_check if _get_block_count(_local_region.memory[idx]) >= blocks_needed {
|
||||
break infinite
|
||||
}
|
||||
}
|
||||
sync.atomic_store_explicit(&_local_region.hdr.local_addr, &_local_region, .Release)
|
||||
_local_region, back_idx = _region_retrieve_with_space(blocks_needed, local_region_idx, back_idx)
|
||||
}
|
||||
user_ptr, used := _region_get_block(_local_region, idx, blocks_needed)
|
||||
|
||||
sync.atomic_sub_explicit(&_local_region.hdr.free_blocks, used + 1, .Release)
|
||||
|
||||
// If this memory was ever used before, it now needs to be zero'd.
|
||||
if idx < _local_region.hdr.last_used {
|
||||
mem.zero(user_ptr, int(used) * BLOCK_SIZE)
|
||||
} else {
|
||||
_local_region.hdr.last_used = idx + used
|
||||
}
|
||||
|
||||
return user_ptr
|
||||
}
|
||||
|
||||
heap_resize :: proc(old_memory: rawptr, new_size: int) -> rawptr #no_bounds_check {
|
||||
alloc := _get_allocation_header(old_memory)
|
||||
if alloc.requested & IS_DIRECT_MMAP > 0 {
|
||||
return _direct_mmap_resize(alloc, new_size)
|
||||
}
|
||||
|
||||
if new_size > DIRECT_MMAP_THRESHOLD {
|
||||
return _direct_mmap_from_region(alloc, new_size)
|
||||
}
|
||||
|
||||
return _region_resize(alloc, new_size)
|
||||
}
|
||||
|
||||
heap_free :: proc(memory: rawptr) {
|
||||
alloc := _get_allocation_header(memory)
|
||||
if sync.atomic_load(&alloc.requested) & IS_DIRECT_MMAP == IS_DIRECT_MMAP {
|
||||
_direct_mmap_free(alloc)
|
||||
return
|
||||
}
|
||||
|
||||
assert(alloc.free_idx == NOT_FREE)
|
||||
|
||||
_region_find_and_assign_local(alloc)
|
||||
_region_local_free(alloc)
|
||||
sync.atomic_store_explicit(&_local_region.hdr.local_addr, &_local_region, .Release)
|
||||
}
|
||||
|
||||
//
|
||||
// Regions
|
||||
//
|
||||
_new_region :: proc() -> ^Region #no_bounds_check {
|
||||
ptr, errno := linux.mmap(0, uint(SIZE_OF_REGION), MMAP_PROT, MMAP_FLAGS, -1, 0)
|
||||
if errno != .NONE {
|
||||
return nil
|
||||
}
|
||||
new_region := (^Region)(ptr)
|
||||
|
||||
new_region.hdr.local_addr = CURRENTLY_ACTIVE
|
||||
new_region.hdr.reset_addr = &_local_region
|
||||
|
||||
free_list_blocks := _round_up_to_nearest(FREE_LIST_DEFAULT_CAP, FREE_LIST_ENTRIES_PER_BLOCK)
|
||||
_region_assign_free_list(new_region, &new_region.memory[1], u16(free_list_blocks) * FREE_LIST_ENTRIES_PER_BLOCK)
|
||||
|
||||
// + 2 to account for free_list's allocation header
|
||||
first_user_block := len(new_region.hdr.free_list) / FREE_LIST_ENTRIES_PER_BLOCK + 2
|
||||
|
||||
// first allocation header (this is a free list)
|
||||
new_region.memory[0].next = u16(first_user_block)
|
||||
new_region.memory[0].free_idx = NOT_FREE
|
||||
new_region.memory[first_user_block].idx = u16(first_user_block)
|
||||
new_region.memory[first_user_block].next = BLOCKS_PER_REGION - 1
|
||||
|
||||
// add the first user block to the free list
|
||||
new_region.hdr.free_list[0] = u16(first_user_block)
|
||||
new_region.hdr.free_list_len = 1
|
||||
new_region.hdr.free_blocks = _get_block_count(new_region.memory[first_user_block]) + 1
|
||||
|
||||
for r := sync.atomic_compare_exchange_strong(&global_regions, nil, new_region);
|
||||
r != nil;
|
||||
r = sync.atomic_compare_exchange_strong(&r.hdr.next_region, nil, new_region) {}
|
||||
|
||||
return new_region
|
||||
}
|
||||
|
||||
_region_resize :: proc(alloc: ^Allocation_Header, new_size: int, alloc_is_free_list: bool = false) -> rawptr #no_bounds_check {
|
||||
assert(alloc.free_idx == NOT_FREE)
|
||||
|
||||
old_memory := mem.ptr_offset(alloc, 1)
|
||||
|
||||
old_block_count := _get_block_count(alloc^)
|
||||
new_block_count := u16(
|
||||
max(MINIMUM_BLOCK_COUNT, _round_up_to_nearest(new_size, BLOCK_SIZE) / BLOCK_SIZE),
|
||||
)
|
||||
if new_block_count < old_block_count {
|
||||
if new_block_count - old_block_count >= MINIMUM_BLOCK_COUNT {
|
||||
_region_find_and_assign_local(alloc)
|
||||
_region_segment(_local_region, alloc, new_block_count, alloc.free_idx)
|
||||
new_block_count = _get_block_count(alloc^)
|
||||
sync.atomic_store_explicit(&_local_region.hdr.local_addr, &_local_region, .Release)
|
||||
}
|
||||
// need to zero anything within the new block that that lies beyond new_size
|
||||
extra_bytes := int(new_block_count * BLOCK_SIZE) - new_size
|
||||
extra_bytes_ptr := mem.ptr_offset((^u8)(alloc), new_size + BLOCK_SIZE)
|
||||
mem.zero(extra_bytes_ptr, extra_bytes)
|
||||
return old_memory
|
||||
}
|
||||
|
||||
if !alloc_is_free_list {
|
||||
_region_find_and_assign_local(alloc)
|
||||
}
|
||||
defer if !alloc_is_free_list {
|
||||
sync.atomic_store_explicit(&_local_region.hdr.local_addr, &_local_region, .Release)
|
||||
}
|
||||
|
||||
// First, let's see if we can grow in place.
|
||||
if alloc.next != BLOCKS_PER_REGION - 1 && _local_region.memory[alloc.next].free_idx != NOT_FREE {
|
||||
next_alloc := _local_region.memory[alloc.next]
|
||||
total_available := old_block_count + _get_block_count(next_alloc) + 1
|
||||
if total_available >= new_block_count {
|
||||
alloc.next = next_alloc.next
|
||||
_local_region.memory[alloc.next].prev = alloc.idx
|
||||
if total_available - new_block_count > BLOCK_SEGMENT_THRESHOLD {
|
||||
_region_segment(_local_region, alloc, new_block_count, next_alloc.free_idx)
|
||||
} else {
|
||||
_region_free_list_remove(_local_region, next_alloc.free_idx)
|
||||
}
|
||||
mem.zero(&_local_region.memory[next_alloc.idx], int(alloc.next - next_alloc.idx) * BLOCK_SIZE)
|
||||
_local_region.hdr.last_used = max(alloc.next, _local_region.hdr.last_used)
|
||||
_local_region.hdr.free_blocks -= (_get_block_count(alloc^) - old_block_count)
|
||||
if alloc_is_free_list {
|
||||
_region_assign_free_list(_local_region, old_memory, _get_block_count(alloc^))
|
||||
}
|
||||
return old_memory
|
||||
}
|
||||
}
|
||||
|
||||
// If we made it this far, we need to resize, copy, zero and free.
|
||||
region_iter := _local_region
|
||||
local_region_idx := _region_get_local_idx()
|
||||
back_idx := -1
|
||||
idx: u16
|
||||
infinite: for {
|
||||
for i := 0; i < int(region_iter.hdr.free_list_len); i += 1 {
|
||||
idx = region_iter.hdr.free_list[i]
|
||||
if _get_block_count(region_iter.memory[idx]) >= new_block_count {
|
||||
break infinite
|
||||
}
|
||||
}
|
||||
if region_iter != _local_region {
|
||||
sync.atomic_store_explicit(
|
||||
®ion_iter.hdr.local_addr,
|
||||
region_iter.hdr.reset_addr,
|
||||
.Release,
|
||||
)
|
||||
}
|
||||
region_iter, back_idx = _region_retrieve_with_space(new_block_count, local_region_idx, back_idx)
|
||||
}
|
||||
if region_iter != _local_region {
|
||||
sync.atomic_store_explicit(
|
||||
®ion_iter.hdr.local_addr,
|
||||
region_iter.hdr.reset_addr,
|
||||
.Release,
|
||||
)
|
||||
}
|
||||
|
||||
// copy from old memory
|
||||
new_memory, used_blocks := _region_get_block(region_iter, idx, new_block_count)
|
||||
mem.copy(new_memory, old_memory, int(old_block_count * BLOCK_SIZE))
|
||||
|
||||
// zero any new memory
|
||||
addon_section := mem.ptr_offset((^Allocation_Header)(new_memory), old_block_count)
|
||||
new_blocks := used_blocks - old_block_count
|
||||
mem.zero(addon_section, int(new_blocks) * BLOCK_SIZE)
|
||||
|
||||
region_iter.hdr.free_blocks -= (used_blocks + 1)
|
||||
|
||||
// Set free_list before freeing.
|
||||
if alloc_is_free_list {
|
||||
_region_assign_free_list(_local_region, new_memory, used_blocks)
|
||||
}
|
||||
|
||||
// free old memory
|
||||
_region_local_free(alloc)
|
||||
return new_memory
|
||||
}
|
||||
|
||||
_region_local_free :: proc(alloc: ^Allocation_Header) #no_bounds_check {
|
||||
alloc := alloc
|
||||
add_to_free_list := true
|
||||
|
||||
idx := sync.atomic_load(&alloc.idx)
|
||||
prev := sync.atomic_load(&alloc.prev)
|
||||
next := sync.atomic_load(&alloc.next)
|
||||
block_count := next - idx - 1
|
||||
free_blocks := sync.atomic_load(&_local_region.hdr.free_blocks) + block_count + 1
|
||||
sync.atomic_store_explicit(&_local_region.hdr.free_blocks, free_blocks, .Release)
|
||||
|
||||
// try to merge with prev
|
||||
if idx > 0 && sync.atomic_load(&_local_region.memory[prev].free_idx) != NOT_FREE {
|
||||
sync.atomic_store_explicit(&_local_region.memory[prev].next, next, .Release)
|
||||
_local_region.memory[next].prev = prev
|
||||
alloc = &_local_region.memory[prev]
|
||||
add_to_free_list = false
|
||||
}
|
||||
|
||||
// try to merge with next
|
||||
if next < BLOCKS_PER_REGION - 1 && sync.atomic_load(&_local_region.memory[next].free_idx) != NOT_FREE {
|
||||
old_next := next
|
||||
sync.atomic_store_explicit(&alloc.next, sync.atomic_load(&_local_region.memory[old_next].next), .Release)
|
||||
|
||||
sync.atomic_store_explicit(&_local_region.memory[next].prev, idx, .Release)
|
||||
|
||||
if add_to_free_list {
|
||||
sync.atomic_store_explicit(&_local_region.hdr.free_list[_local_region.memory[old_next].free_idx], idx, .Release)
|
||||
sync.atomic_store_explicit(&alloc.free_idx, _local_region.memory[old_next].free_idx, .Release)
|
||||
} else {
|
||||
// NOTE: We have aleady merged with prev, and now merged with next.
|
||||
// Now, we are actually going to remove from the free_list.
|
||||
_region_free_list_remove(_local_region, _local_region.memory[old_next].free_idx)
|
||||
}
|
||||
add_to_free_list = false
|
||||
}
|
||||
|
||||
// This is the only place where anything is appended to the free list.
|
||||
if add_to_free_list {
|
||||
fl := _local_region.hdr.free_list
|
||||
fl_len := sync.atomic_load(&_local_region.hdr.free_list_len)
|
||||
sync.atomic_store_explicit(&alloc.free_idx, fl_len, .Release)
|
||||
fl[alloc.free_idx] = idx
|
||||
sync.atomic_store_explicit(&_local_region.hdr.free_list_len, fl_len + 1, .Release)
|
||||
if int(fl_len + 1) == len(fl) {
|
||||
free_alloc := _get_allocation_header(mem.raw_data(_local_region.hdr.free_list))
|
||||
_region_resize(free_alloc, len(fl) * 2 * size_of(fl[0]), true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_region_assign_free_list :: proc(region: ^Region, memory: rawptr, blocks: u16) {
|
||||
raw_free_list := transmute(mem.Raw_Slice)region.hdr.free_list
|
||||
raw_free_list.len = int(blocks) * FREE_LIST_ENTRIES_PER_BLOCK
|
||||
raw_free_list.data = memory
|
||||
region.hdr.free_list = transmute([]u16)(raw_free_list)
|
||||
}
|
||||
|
||||
_region_retrieve_with_space :: proc(blocks: u16, local_idx: int = -1, back_idx: int = -1) -> (^Region, int) {
|
||||
r: ^Region
|
||||
idx: int
|
||||
for r = sync.atomic_load(&global_regions); r != nil; r = r.hdr.next_region {
|
||||
if idx == local_idx || idx < back_idx || sync.atomic_load(&r.hdr.free_blocks) < blocks {
|
||||
idx += 1
|
||||
continue
|
||||
}
|
||||
idx += 1
|
||||
local_addr: ^^Region = sync.atomic_load(&r.hdr.local_addr)
|
||||
if local_addr != CURRENTLY_ACTIVE {
|
||||
res := sync.atomic_compare_exchange_strong_explicit(
|
||||
&r.hdr.local_addr,
|
||||
local_addr,
|
||||
CURRENTLY_ACTIVE,
|
||||
.Acquire,
|
||||
.Relaxed,
|
||||
)
|
||||
if res == local_addr {
|
||||
r.hdr.reset_addr = local_addr
|
||||
return r, idx
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return _new_region(), idx
|
||||
}
|
||||
|
||||
_region_retrieve_from_addr :: proc(addr: rawptr) -> ^Region {
|
||||
r: ^Region
|
||||
for r = global_regions; r != nil; r = r.hdr.next_region {
|
||||
if _region_contains_mem(r, addr) {
|
||||
return r
|
||||
}
|
||||
}
|
||||
unreachable()
|
||||
}
|
||||
|
||||
_region_get_block :: proc(region: ^Region, idx, blocks_needed: u16) -> (rawptr, u16) #no_bounds_check {
|
||||
alloc := ®ion.memory[idx]
|
||||
|
||||
assert(alloc.free_idx != NOT_FREE)
|
||||
assert(alloc.next > 0)
|
||||
|
||||
block_count := _get_block_count(alloc^)
|
||||
if block_count - blocks_needed > BLOCK_SEGMENT_THRESHOLD {
|
||||
_region_segment(region, alloc, blocks_needed, alloc.free_idx)
|
||||
} else {
|
||||
_region_free_list_remove(region, alloc.free_idx)
|
||||
}
|
||||
|
||||
alloc.free_idx = NOT_FREE
|
||||
return mem.ptr_offset(alloc, 1), _get_block_count(alloc^)
|
||||
}
|
||||
|
||||
_region_segment :: proc(region: ^Region, alloc: ^Allocation_Header, blocks, new_free_idx: u16) #no_bounds_check {
|
||||
old_next := alloc.next
|
||||
alloc.next = alloc.idx + blocks + 1
|
||||
region.memory[old_next].prev = alloc.next
|
||||
|
||||
// Initialize alloc.next allocation header here.
|
||||
region.memory[alloc.next].prev = alloc.idx
|
||||
region.memory[alloc.next].next = old_next
|
||||
region.memory[alloc.next].idx = alloc.next
|
||||
region.memory[alloc.next].free_idx = new_free_idx
|
||||
|
||||
// Replace our original spot in the free_list with new segment.
|
||||
region.hdr.free_list[new_free_idx] = alloc.next
|
||||
}
|
||||
|
||||
_region_get_local_idx :: proc() -> int {
|
||||
idx: int
|
||||
for r := sync.atomic_load(&global_regions); r != nil; r = r.hdr.next_region {
|
||||
if r == _local_region {
|
||||
return idx
|
||||
}
|
||||
idx += 1
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
_region_find_and_assign_local :: proc(alloc: ^Allocation_Header) {
|
||||
// Find the region that contains this memory
|
||||
if !_region_contains_mem(_local_region, alloc) {
|
||||
_local_region = _region_retrieve_from_addr(alloc)
|
||||
}
|
||||
|
||||
// At this point, _local_region is set correctly. Spin until acquire
|
||||
res := CURRENTLY_ACTIVE
|
||||
|
||||
for res == CURRENTLY_ACTIVE {
|
||||
res = sync.atomic_compare_exchange_strong_explicit(
|
||||
&_local_region.hdr.local_addr,
|
||||
&_local_region,
|
||||
CURRENTLY_ACTIVE,
|
||||
.Acquire,
|
||||
.Relaxed,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
_region_contains_mem :: proc(r: ^Region, memory: rawptr) -> bool #no_bounds_check {
|
||||
if r == nil {
|
||||
return false
|
||||
}
|
||||
mem_int := uintptr(memory)
|
||||
return mem_int >= uintptr(&r.memory[0]) && mem_int <= uintptr(&r.memory[BLOCKS_PER_REGION - 1])
|
||||
}
|
||||
|
||||
_region_free_list_remove :: proc(region: ^Region, free_idx: u16) #no_bounds_check {
|
||||
// pop, swap and update allocation hdr
|
||||
if n := region.hdr.free_list_len - 1; free_idx != n {
|
||||
region.hdr.free_list[free_idx] = sync.atomic_load(®ion.hdr.free_list[n])
|
||||
alloc_idx := region.hdr.free_list[free_idx]
|
||||
sync.atomic_store_explicit(®ion.memory[alloc_idx].free_idx, free_idx, .Release)
|
||||
}
|
||||
region.hdr.free_list_len -= 1
|
||||
}
|
||||
|
||||
//
|
||||
// Direct mmap
|
||||
//
|
||||
_direct_mmap_alloc :: proc(size: int) -> rawptr {
|
||||
mmap_size := _round_up_to_nearest(size + BLOCK_SIZE, PAGE_SIZE)
|
||||
new_allocation, errno := linux.mmap(0, uint(mmap_size), MMAP_PROT, MMAP_FLAGS, -1, 0)
|
||||
if errno != .NONE {
|
||||
return nil
|
||||
}
|
||||
|
||||
alloc := (^Allocation_Header)(uintptr(new_allocation))
|
||||
alloc.requested = u64(size) // NOTE: requested = requested size
|
||||
alloc.requested += IS_DIRECT_MMAP
|
||||
return rawptr(mem.ptr_offset(alloc, 1))
|
||||
}
|
||||
|
||||
_direct_mmap_resize :: proc(alloc: ^Allocation_Header, new_size: int) -> rawptr {
|
||||
old_requested := int(alloc.requested & REQUESTED_MASK)
|
||||
old_mmap_size := _round_up_to_nearest(old_requested + BLOCK_SIZE, PAGE_SIZE)
|
||||
new_mmap_size := _round_up_to_nearest(new_size + BLOCK_SIZE, PAGE_SIZE)
|
||||
if int(new_mmap_size) < MMAP_TO_REGION_SHRINK_THRESHOLD {
|
||||
return _direct_mmap_to_region(alloc, new_size)
|
||||
} else if old_requested == new_size {
|
||||
return mem.ptr_offset(alloc, 1)
|
||||
}
|
||||
|
||||
new_allocation, errno := linux.mremap(alloc, uint(old_mmap_size), uint(new_mmap_size), {.MAYMOVE})
|
||||
if errno != .NONE {
|
||||
return nil
|
||||
}
|
||||
|
||||
new_header := (^Allocation_Header)(uintptr(new_allocation))
|
||||
new_header.requested = u64(new_size)
|
||||
new_header.requested += IS_DIRECT_MMAP
|
||||
|
||||
if new_mmap_size > old_mmap_size {
|
||||
// new section may not be pointer aligned, so cast to ^u8
|
||||
new_section := mem.ptr_offset((^u8)(new_header), old_requested + BLOCK_SIZE)
|
||||
mem.zero(new_section, new_mmap_size - old_mmap_size)
|
||||
}
|
||||
return mem.ptr_offset(new_header, 1)
|
||||
|
||||
}
|
||||
|
||||
_direct_mmap_from_region :: proc(alloc: ^Allocation_Header, new_size: int) -> rawptr {
|
||||
new_memory := _direct_mmap_alloc(new_size)
|
||||
if new_memory != nil {
|
||||
old_memory := mem.ptr_offset(alloc, 1)
|
||||
mem.copy(new_memory, old_memory, int(_get_block_count(alloc^)) * BLOCK_SIZE)
|
||||
}
|
||||
_region_find_and_assign_local(alloc)
|
||||
_region_local_free(alloc)
|
||||
sync.atomic_store_explicit(&_local_region.hdr.local_addr, &_local_region, .Release)
|
||||
return new_memory
|
||||
}
|
||||
|
||||
_direct_mmap_to_region :: proc(alloc: ^Allocation_Header, new_size: int) -> rawptr {
|
||||
new_memory := heap_alloc(new_size)
|
||||
if new_memory != nil {
|
||||
mem.copy(new_memory, mem.ptr_offset(alloc, -1), new_size)
|
||||
_direct_mmap_free(alloc)
|
||||
}
|
||||
return new_memory
|
||||
}
|
||||
|
||||
_direct_mmap_free :: proc(alloc: ^Allocation_Header) {
|
||||
requested := int(alloc.requested & REQUESTED_MASK)
|
||||
mmap_size := _round_up_to_nearest(requested + BLOCK_SIZE, PAGE_SIZE)
|
||||
linux.munmap(alloc, uint(mmap_size))
|
||||
}
|
||||
|
||||
//
|
||||
// Util
|
||||
//
|
||||
|
||||
_get_block_count :: #force_inline proc(alloc: Allocation_Header) -> u16 {
|
||||
return alloc.next - alloc.idx - 1
|
||||
}
|
||||
|
||||
_get_allocation_header :: #force_inline proc(raw_mem: rawptr) -> ^Allocation_Header {
|
||||
return mem.ptr_offset((^Allocation_Header)(raw_mem), -1)
|
||||
}
|
||||
|
||||
_round_up_to_nearest :: #force_inline proc(size, round: int) -> int {
|
||||
return (size-1) + round - (size-1) % round
|
||||
}
|
||||
import "base:runtime"
|
||||
|
||||
_heap_allocator_proc :: runtime.heap_allocator_proc
|
||||
|
||||
Reference in New Issue
Block a user