From b056b9e1309c317fdf3480582a06ab7f2ed7af7b Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Tue, 6 May 2025 16:09:42 -0400 Subject: [PATCH] Rewrite the heap allocator This is a total rewrite after having experimented with a variety of techniques including a dual-allocator strategy for small and large size classes where the large class allocator had a coalescing mechanism. This allocator design is much closer to `mimalloc` in spirit and performs almost twice as fast as the original bitmap-based design. Additionally, several bugs have been fixed. The most important one being the lock-free synchronization method used for remote frees. This design uses a single atomic pointer that is doubly-tagged to pass remote frees either to the heap or to the slab, depending on the ownership status. --- base/runtime/heap_allocator.odin | 2 +- base/runtime/heap_allocator_control.odin | 125 +- .../heap_allocator_implementation.odin | 3041 ++++++++--------- base/runtime/heap_allocator_info.odin | 256 +- tests/heap_allocator/test_bench.odin | 436 +-- 5 files changed, 1604 insertions(+), 2256 deletions(-) diff --git a/base/runtime/heap_allocator.odin b/base/runtime/heap_allocator.odin index 7fc1c45c6..ad45c37f4 100644 --- a/base/runtime/heap_allocator.odin +++ b/base/runtime/heap_allocator.odin @@ -20,7 +20,7 @@ heap_allocator_proc :: proc( old_size: int, loc := #caller_location, ) -> ([]byte, Allocator_Error) { - assert(alignment <= HEAP_MAX_ALIGNMENT, "Heap allocation alignment beyond HEAP_MAX_ALIGNMENT bytes is not supported.", loc = loc) + assert(alignment <= ODIN_HEAP_MAX_ALIGNMENT, "Heap allocation alignment beyond ODIN_HEAP_MAX_ALIGNMENT bytes is not supported.", loc = loc) assert(alignment >= 0, "Alignment must be greater than or equal to zero.", loc = loc) switch mode { case .Alloc: diff --git a/base/runtime/heap_allocator_control.odin b/base/runtime/heap_allocator_control.odin index 4f5ae68b4..b805096cf 100644 --- a/base/runtime/heap_allocator_control.odin +++ b/base/runtime/heap_allocator_control.odin @@ -6,87 +6,84 @@ package runtime import "base:intrinsics" /* -Merge all remote frees then free as many slabs as possible. - -This bypasses any heuristics that keep slabs setup. - -Returns true if the superpage was emptied and freed. +Reduce the amount of dynamically allocated memory held by the current thread as much as possible. */ -@(private) -compact_superpage :: proc "contextless" (superpage: ^Heap_Superpage) -> (freed: bool) { - for i := 0; i < HEAP_SLAB_COUNT; /**/ { - slab := heap_superpage_index_slab(superpage, i) +compact_local_heap :: proc "contextless" () { + if local_heap == nil { + return + } - if slab.bin_size > HEAP_MAX_BIN_SIZE { - // Skip contiguous slabs. - i += heap_slabs_needed_for_size(slab.bin_size) - } else { - i += 1 - if slab.bin_size == 0 { - continue - } - } + heap_merge_remote_free_list() - slab_is_cached := slab.free_bins > 0 - heap_merge_remote_frees(slab) + for segment := local_heap.segments; segment != nil; /**/ { + next := segment.next_segment - if slab.free_bins == slab.max_bins { - if slab.bin_size > HEAP_MAX_BIN_SIZE { - heap_free_wide_slab(superpage, slab) - } else { - if slab_is_cached { - heap_cache_remove_slab(slab, heap_bin_size_to_rank(slab.bin_size)) + segment_will_free_itself := segment.may_return + free_slabs := segment.free_slabs + max_slabs := len(segment.slabs) + + for i in 0.. 0 && slab.free_bins == slab.max_bins { + free_slabs += 1 + heap_free_slab(segment, slab) + if free_slabs == max_slabs { + // We must break now, as the segment's memory could have + // been returned to the operating system and we may + // continue iterating over invalid memory. + break } - heap_free_slab(superpage, slab) } } - } - if superpage.free_slabs == HEAP_SLAB_COUNT && !superpage.cache_block.in_use { - heap_free_superpage(superpage) - freed = true - } - return -} - -/* -Merge all remote frees then free as many slabs and superpages as possible. - -This bypasses any heuristics that keep slabs setup. -*/ -compact_heap :: proc "contextless" () { - superpage := local_heap - for { - if superpage == nil { - return + // We check it this way because `heap_free_slab` will automatically + // free the segment if the conditions are right, otherwise we need to + // do it. + if free_slabs == max_slabs && !segment_will_free_itself { + heap_free_segment(segment) } - next_superpage := superpage.next - compact_superpage(superpage) - superpage = next_superpage + + segment = next } } /* -Free any empty superpages in the orphanage. - -This procedure assumes there won't ever be more than 128 superpages in the -orphanage. This limitation is due to the avoidance of heap allocation. +Adopt all empty segments in the orphanage and release them back to the operating system. */ -compact_heap_orphanage :: proc "contextless" () { - // First, try to empty the orphanage so that we can evaluate each superpage. - buffer: [128]^Heap_Superpage - for &b in buffer { - b = heap_pop_orphan() - if b == nil { +heap_release_empty_orphans :: proc "contextless" () { + segment: ^Heap_Segment + + // First, take control of the linked list by replacing it with a nil + // pointer and a zero count. + old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&heap_orphanage.empty, .Relaxed) + for { + count := old_head.pointer & ODIN_HEAP_ORPHANAGE_COUNT_BITS + untagged_head := uintptr(old_head.pointer) & ~uintptr(ODIN_HEAP_ORPHANAGE_COUNT_BITS) + + segment = cast(^Heap_Segment)uintptr(untagged_head) + if segment == nil { + assert_contextless(count == 0, "The heap allocator saw a nil pointer on the orphanage for empty segments but the count was not zero.") break } + + new_head := Tagged_Pointer{ + pointer = 0, // nil pointer with zero count + version = old_head.version + 1, + } + + old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)&heap_orphanage.empty, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) + if swapped { + intrinsics.atomic_store_explicit(&segment.next_segment, nil, .Release) + break + } + old_head = transmute(Tagged_Pointer)old_head_ } - // Next, compact each superpage and push it back to the orphanage if it was - // not freed. - for superpage in buffer { - if !compact_superpage(superpage) { - heap_push_orphan(superpage) - } + // Now walk the list of segments and release them. + for segment != nil { + next := segment.next_segment + assert_contextless(segment.free_slabs == len(segment.slabs), "The heap allocator found a segment in the orphanage that should have been empty.") + free_virtual_memory(segment, segment.size) + segment = next } } diff --git a/base/runtime/heap_allocator_implementation.odin b/base/runtime/heap_allocator_implementation.odin index 1077aa9c1..8336c01bb 100644 --- a/base/runtime/heap_allocator_implementation.odin +++ b/base/runtime/heap_allocator_implementation.odin @@ -16,1256 +16,1160 @@ This is the dynamic heap allocator for the Odin runtime. - Thread-local heaps: There is no allocator-induced false sharing. - Global storage for unused memory: When a thread finishes cleanly, its memory - is sent to a global storage where other threads can use it. + is sent to the orphanage where other threads can use it. When an entire + segment is freed, the orphanage will hold some in reserve to prevent too many + requests to the operating system for virtual memory. -- Headerless: All bin-sized allocations (any power of two, less than or equal - to 32KiB, by default) consume no extra space, and as a result of being tightly - packed, enhance performance with cache locality for programs. +- Headerless: Except for the heap metadata needed to support them, each + allocation consumes no extra space, and as a result of being tightly packed, + performance is enhanced with cache locality for programs. + +- Double-Free Checking: In debug mode, the allocator will take extra memory to + keep track of double frees in a per-Slab bitmap. **Terminology** -- Bin: an allocation of a fixed size, shared with others of the same size category. +- Segment: a single contiguous allocation from the operating system that + contains metadata about its allocations and is divided into at least one Slab. -- Slab: a fixed-size block within a Superpage that is divided into a constant +- Slab: a fixed-size block within a Segment that is divided into a constant number of Bins at runtime based on the needs of the program. -- Sector: a variable-sized bitmap, used to track whether a Bin is free or not. +- Bin: an allocation of a fixed power-of-two size, shared with others of the + same size category. These fixed size categories are called ranks. -**Allocation Categories** +**Size Classes** -- Huge: anything in excess of 3/4ths of a Superpage. -- Slab-wide: anything in excess of the largest Bin size. -- Bin: anything less than or equal to the largest bin size. +Segments are divided based on the initial allocation request which causes them +to be needed. + +For example, an allocation of 8 bytes will cause a Segment of Small Slabs to be +made to support it, and an allocation of 128KiB will cause a Segment of Large +Slabs to be made. + +Each Segment is subdivided to support as many Slabs as can be held, except for +allocations over 512KiB; those are given their own single-Slab Segment and are +returned to the operating system immediately upon freeing. + +- Small: Allocations <= 8KiB are placed into Small-subdivided Segments. +- Large: Allocations <= 512KiB are placed into Large-subdivided Segments. +- Huge: Allocations > 512KiB are given their own single-Slab Segment. */ // // Tunables // -// NOTE: Adjusting this constant by itself is rarely enough; `HEAP_SUPERPAGE_CACHE_RATIO` -// will have to be changed as well. -HEAP_SLAB_SIZE :: #config(ODIN_HEAP_SLAB_SIZE, 64 * Kilobyte) -HEAP_MAX_BIN_SIZE :: #config(ODIN_HEAP_MAX_BIN_SIZE, HEAP_SLAB_SIZE / 2) -HEAP_MIN_BIN_SIZE :: #config(ODIN_HEAP_MIN_BIN_SIZE, 8 * Byte) -HEAP_MAX_EMPTY_ORPHANED_SUPERPAGES :: #config(ODIN_HEAP_MAX_EMPTY_ORPHANED_SUPERPAGES, 3) -HEAP_SUPERPAGE_CACHE_RATIO :: #config(ODIN_HEAP_SUPERPAGE_CACHE_RATIO, 20) -HEAP_PANIC_ON_DOUBLE_FREE :: #config(ODIN_HEAP_PANIC_ON_DOUBLE_FREE, true) +/* +`ODIN_HEAP_SEGMENT_SIZE_OVERRIDE` controls how many bytes are allocated for each heap segment. + +The default value of zero causes the allocator to use the superpage size of operating system. +*/ +ODIN_HEAP_SEGMENT_SIZE_OVERRIDE :: #config(ODIN_HEAP_SEGMENT_SIZE_OVERRIDE, 0 /* bytes */) + +/* +`ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS` controls how many empty segments are kept on +hand for re-use instead of being immediately returned to the operating system. +*/ +ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS :: #config(ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS, 5 /* segments */) + +/* +`ODIN_HEAP_DEBUG_LEVEL` controls exactly how much debug checking the allocator will +do. The levels are ordered from increasing levels of computational complexity +and the higher the level, the slower the program will run. +*/ +ODIN_HEAP_DEBUG_LEVEL :: Heap_Debug_Level(HEAP_DEBUG_LEVEL) +@(private="file") +HEAP_DEBUG_LEVEL :: #config(ODIN_HEAP_DEBUG_LEVEL, 3 when ODIN_DEBUG else 0) + +/* +`ODIN_HEAP_MIN_BIN_SIZE` and `ODIN_HEAP_MAX_BIN_SIZE` control the range of the size of +the bins in power-of-two intervals from each other as an inclusive range. + +Below `ODIN_HEAP_MIN_BIN_SIZE`, all requests are rounded up to the minimum. +Beyond `ODIN_HEAP_MAX_BIN_SIZE`, all requests are given their own specifically-sized allocation. +*/ +ODIN_HEAP_MIN_BIN_SIZE :: #config(ODIN_HEAP_MIN_BIN_SIZE, 8 * Byte) +ODIN_HEAP_MAX_BIN_SIZE :: #config(ODIN_HEAP_MIN_BIN_SIZE, 512 * Kilobyte) // [n..=m] inclusive range + +/* +`ODIN_HEAP_MAX_ALIGNMENT` controls the maximum supported alignment. +*/ +ODIN_HEAP_MAX_ALIGNMENT :: #config(ODIN_HEAP_MAX_ALIGNMENT, 64 * Byte) + +/* +`ODIN_HEAP_SMALL_SLAB_SIZE` controls the cut-off for Segments with Small Slabs. +Any allocation below `ODIN_HEAP_SMALL_BIN_MAX` will be placed into Slabs of this size. + +Beyond that, allocations are placed into Large Slabs that consume an entire +Segment for the power-of-two size request. For example, an allocation of 16KiB +will result in a Segment that has been partitioned with only one Slab but may +use the entire width of the Slab space for any allocation that rounds to 16KiB. +*/ +ODIN_HEAP_SMALL_SLAB_SIZE :: #config(ODIN_HEAP_SMALL_SLAB_SIZE, 64 * Kilobyte) +ODIN_HEAP_SMALL_BIN_MAX :: #config(ODIN_HEAP_SMALL_BIN_MAX, 8 * Kilobyte) // [0..=m] inclusive range // // Constants // -HEAP_MAX_BIN_SHIFT :: intrinsics.constant_log2(HEAP_MAX_BIN_SIZE) -HEAP_MIN_BIN_SHIFT :: intrinsics.constant_log2(HEAP_MIN_BIN_SIZE) -HEAP_BIN_RANKS :: 1 + HEAP_MAX_BIN_SHIFT - HEAP_MIN_BIN_SHIFT -HEAP_HUGE_ALLOCATION_BOOK_KEEPING :: size_of(int) + HEAP_MAX_ALIGNMENT -HEAP_HUGE_ALLOCATION_THRESHOLD :: SUPERPAGE_SIZE / 4 * 3 -HEAP_MAX_ALIGNMENT :: 64 * Byte -HEAP_CACHE_SLAB_MAP_STRIDE :: HEAP_SUPERPAGE_CACHE_RATIO * HEAP_SLAB_COUNT -HEAP_SECTOR_TYPES :: 2 // { local_free, remote_free } -HEAP_SLAB_ALLOCATION_BOOK_KEEPING :: size_of(Heap_Slab) + HEAP_SECTOR_TYPES * size_of(uint) + HEAP_MAX_ALIGNMENT -HEAP_SLAB_COUNT :: SUPERPAGE_SIZE / HEAP_SLAB_SIZE - 1 +ODIN_HEAP_MIN_BIN_SHIFT :: intrinsics.constant_log2(ODIN_HEAP_MIN_BIN_SIZE) +ODIN_HEAP_MAX_BIN_SHIFT :: intrinsics.constant_log2(ODIN_HEAP_MAX_BIN_SIZE) +ODIN_HEAP_BIN_RANKS :: 1 + ODIN_HEAP_MAX_BIN_SHIFT - ODIN_HEAP_MIN_BIN_SHIFT -@(private="file") INTEGER_BITS :: 8 * size_of(int) -@(private="file") SECTOR_BITS :: 8 * size_of(uint) +// This mask is used to store an atomic count within a `Tagged_Pointer` to +// limit the number of empty Segments sent into the orphanage. +ODIN_HEAP_ORPHANAGE_COUNT_BITS :: 0xFFFF + +@(private) +HEAP_FREE_LIST_CLOSED :: 0x01 + +Heap_Debug_Level :: enum { + // No extra work is done beyond the sanity checking in the assertion statements. + None = 0, + + // Some allocation statistics are monitored in real-time. + Statistics = 1, + + // This level causes an extra bitmap to be allocated outside of the space + // used for the heap and its slabs. Freed bins will be tracked there to + // ensure no double frees occur. + Double_Free = 2, + + // This level does extra checking using the `double_free_tracker` bitmap to + // make sure that addresses pulled from a slab's free list exist within the + // slab and are truly free. + // + // Additionally, all addresses are XOR'd by a key that is specific to the + // slab that owns it or a global key for remote frees. This is to prevent + // overwriting a free list entry with an address that would be a valid + // pointer but was not meant to be in the free list. + // + // NOTE: This is the default level when `ODIN_DEBUG` is on. + Free_List_Corruption = 3, + + // This level makes sure that each new allocation on an untouched slab is + // completely zero. + Ensure_Zero = 4, + + // This level is very slow, as it has to take a lock and check every + // segment currently allocated to see if the address being freed exists + // within the space of any of them. + // + // Additionally, every heap that exits without freeing all of its memory + // will remain active indefinitely so that the allocator can scan it later + // for valid addresses. + // + // NOTE: The allocator is no longer lock-free at this stage. + Invalid_Free = 5, +} // // Sanity checking // -#assert(HEAP_MAX_BIN_SIZE & (HEAP_MAX_BIN_SIZE-1) == 0, "HEAP_MAX_BIN_SIZE must be a power of two.") -#assert(HEAP_MAX_BIN_SIZE < HEAP_SLAB_SIZE - size_of(Heap_Slab) - HEAP_SECTOR_TYPES * size_of(uint), "HEAP_MAX_BIN_SIZE must be able to fit into one Slab, including its free maps.") -#assert(HEAP_MAX_BIN_SIZE >= HEAP_MIN_BIN_SIZE, "HEAP_MAX_BIN_SIZE must be greater than or equal to HEAP_MIN_BIN_SIZE.") -#assert(HEAP_HUGE_ALLOCATION_THRESHOLD <= SUPERPAGE_SIZE - HEAP_HUGE_ALLOCATION_BOOK_KEEPING, "HEAP_HUGE_ALLOCATION_THRESHOLD must be smaller than a Superpage, with enough space for HEAP_HUGE_ALLOCATION_BOOK_KEEPING.") -#assert(HEAP_MIN_BIN_SIZE & (HEAP_MIN_BIN_SIZE-1) == 0, "HEAP_MIN_BIN_SIZE must be a power of two.") -#assert(HEAP_MAX_EMPTY_ORPHANED_SUPERPAGES >= 0, "HEAP_MAX_EMPTY_ORPHANED_SUPERPAGES must be positive.") -#assert(HEAP_MAX_ALIGNMENT & (HEAP_MAX_ALIGNMENT-1) == 0, "HEAP_MAX_ALIGNMENT must be a power of two.") -#assert(HEAP_SLAB_COUNT > 0, "HEAP_SLAB_COUNT must be greater than zero.") -#assert(HEAP_SLAB_SIZE & (HEAP_SLAB_SIZE-1) == 0, "HEAP_SLAB_SIZE must be a power of two.") -#assert(SUPERPAGE_SIZE >= 2 * HEAP_SLAB_SIZE, "SUPERPAGE_SIZE must be at least twice HEAP_SLAB_SIZE.") -#assert(size_of(Heap_Superpage) < HEAP_SLAB_SIZE, "The Superpage struct must not exceed the size of a Slab.") +#assert(ODIN_HEAP_SEGMENT_SIZE_OVERRIDE & (ODIN_HEAP_SEGMENT_SIZE_OVERRIDE-1) == 0, "ODIN_HEAP_SEGMENT_SIZE_OVERRIDE must be a power of two.") +#assert(ODIN_HEAP_SEGMENT_SIZE_OVERRIDE == 0 || ODIN_HEAP_SEGMENT_SIZE_OVERRIDE > ODIN_HEAP_ORPHANAGE_COUNT_BITS, "ODIN_HEAP_SEGMENT_SIZE_OVERRIDE must be larger than ODIN_HEAP_ORPHANAGE_COUNT_BITS.") +#assert(ODIN_HEAP_MIN_BIN_SIZE & (ODIN_HEAP_MIN_BIN_SIZE-1) == 0, "ODIN_HEAP_MIN_BIN_SIZE must be a power of two.") +#assert(ODIN_HEAP_MAX_BIN_SIZE & (ODIN_HEAP_MAX_BIN_SIZE-1) == 0, "ODIN_HEAP_MAX_BIN_SIZE must be a power of two.") +#assert(ODIN_HEAP_MIN_BIN_SIZE >= size_of(rawptr), "ODIN_HEAP_MIN_BIN_SIZE must be large enough to hold a pointer for the free lists.") +#assert(ODIN_HEAP_MAX_BIN_SIZE >= ODIN_HEAP_MIN_BIN_SIZE, "ODIN_HEAP_MAX_BIN_SIZE must be greater than or equal to ODIN_HEAP_MIN_BIN_SIZE.") +#assert(ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS >= 0, "ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS must be positive.") +#assert(ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS < ODIN_HEAP_ORPHANAGE_COUNT_BITS, "ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS is too great.") +#assert(ODIN_HEAP_MAX_ALIGNMENT & (ODIN_HEAP_MAX_ALIGNMENT-1) == 0, "ODIN_HEAP_MAX_ALIGNMENT must be a power of two.") // // Utility Procedures // -@(require_results, private) -round_to_nearest_power_of_two :: #force_inline proc "contextless" (n: uint) -> int { - assert_contextless(n > 1, "This procedure does not handle the edge case of n < 2.") - return 1 << (INTEGER_BITS - intrinsics.count_leading_zeros(n-1)) -} - -@(require_results) -heap_slabs_needed_for_size :: #force_inline proc "contextless" (size: int) -> (result: int) { - assert_contextless(size > 0) - assert_contextless(size > HEAP_MAX_BIN_SIZE) - size := size - size += HEAP_SLAB_ALLOCATION_BOOK_KEEPING - result = size / HEAP_SLAB_SIZE + (0 if size % HEAP_SLAB_SIZE == 0 else 1) - assert_contextless(result > 0) - assert_contextless(result < HEAP_SLAB_COUNT, "Calculated an overly-large Slab-wide allocation.") - return -} - -@(require_results) -heap_round_to_bin_size :: #force_inline proc "contextless" (n: int) -> int { - if n <= HEAP_MIN_BIN_SIZE { - return HEAP_MIN_BIN_SIZE - } - m := round_to_nearest_power_of_two(uint(n)) - assert_contextless(m & (m-1) == 0, "Internal rounding error.") - return m -} - -@(require_results) -heap_bin_size_to_rank :: #force_inline proc "contextless" (size: int) -> (rank: int) { - // By this point, a size of zero should've been rounded up to HEAP_MIN_BIN_SIZE. - assert_contextless(size > 0, "Size must be greater-than zero.") - assert_contextless(size & (size-1) == 0, "Size must be a power of two.") - rank = intrinsics.count_trailing_zeros(size >> HEAP_MIN_BIN_SHIFT) - assert_contextless(rank <= HEAP_BIN_RANKS, "Bin rank calculated incorrectly; it must be less-than-or-equal-to HEAP_BIN_RANKS.") - return -} - -@(require_results) -find_superpage_from_pointer :: #force_inline proc "contextless" (ptr: rawptr) -> ^Heap_Superpage { - return cast(^Heap_Superpage)(uintptr(ptr) & ~uintptr(SUPERPAGE_SIZE-1)) -} - -@(require_results) -find_slab_from_pointer :: #force_inline proc "contextless" (ptr: rawptr) -> ^Heap_Slab { - return cast(^Heap_Slab)(uintptr(ptr) & ~uintptr(HEAP_SLAB_SIZE-1)) +Heap_Slab_Class :: enum { + Small, // Slabs are `ODIN_HEAP_SMALL_SLAB_SIZE` (64KiB) each. + Large, // One segment-wide (~2MiB) slab. + Huge, // One slab for one allocation, sized specifically for the request. } /* -Get a specific slab by index from a superpage. - -The position deltas are all constant with respect to the origin, hence this -procedure should prove faster than accessing an array of pointers. +Get what Slab size class a `bytes` sized allocation should go to. */ @(require_results) -heap_superpage_index_slab :: #force_inline proc "contextless" (superpage: ^Heap_Superpage, index: int) -> (slab: ^Heap_Slab) { - assert_contextless(index >= 0, "The heap allocator tried to index a negative slab index.") - assert_contextless(index < HEAP_SLAB_COUNT, "The heap allocator tried to index a slab beyond the configured maximum.") - return cast(^Heap_Slab)(uintptr(superpage) + HEAP_SLAB_SIZE * uintptr(1 + index)) +heap_get_size_class :: #force_inline proc "contextless" (bytes: int) -> Heap_Slab_Class { + if bytes <= ODIN_HEAP_SMALL_BIN_MAX { + return .Small + } else if bytes <= ODIN_HEAP_MAX_BIN_SIZE { + return .Large + } else { + return .Huge + } +} + +/* +Allocate a new Segment that may be used to store either Small or Large slabs. +*/ +@(require_results) +heap_allocate_segment :: #force_inline proc "contextless" () -> ^Heap_Segment { + when ODIN_HEAP_SEGMENT_SIZE_OVERRIDE == 0 { + return cast(^Heap_Segment)allocate_virtual_memory_superpage() + } else { + return cast(^Heap_Segment)allocate_virtual_memory_aligned(ODIN_HEAP_SEGMENT_SIZE_OVERRIDE, ODIN_HEAP_SEGMENT_SIZE_OVERRIDE) + } +} + +/* +Get the constant size for all segments. This size also dictates each segment's alignment. +*/ +@(require_results) +heap_get_segment_size :: #force_inline proc "contextless" () -> int { + when ODIN_HEAP_SEGMENT_SIZE_OVERRIDE == 0 { + // TODO: Derive from the OS config. + return SUPERPAGE_SIZE + } else { + return ODIN_HEAP_SEGMENT_SIZE_OVERRIDE + } +} + +/* +Convert a rounded bin size to its integer rank. + +This is used for the `Heap.slabs_by_rank` array of linked lists for fast lookup of slabs by the size they support. + +For example, the default ranks are as follows: + +[Small Slabs] + - 0: 8 + - 1: 16 + - 2: 32 + - 3: 64 + - 4: 128 + - 5: 256 + - 6: 512 + - 7: 1_024 + - 8: 2_048 + - 9: 4_096 + - 10: 8_192 + +[Large Slabs] + - 11: 16_384 + - 12: 32_768 + - 13: 65_536 + - 14: 131_072 + - 15: 262_144 + - 16: 524_288 + +Beyond this size, bins are not ranked; allocations use the Huge class and are made and freed on an as-needed basis. +*/ +@(require_results) +heap_bin_size_to_rank :: proc "contextless" (bin_size: int) -> (rank: int) { + // By this point, a size of zero should've been rounded up to ODIN_HEAP_MIN_BIN_SIZE. + assert_contextless(ODIN_HEAP_MIN_BIN_SIZE <= bin_size && bin_size <= ODIN_HEAP_MAX_BIN_SIZE, "Bin size must be within [ODIN_HEAP_MIN_BINSIZE..=ODIN_HEAP_MAX_BIN_SIZE].") + assert_contextless(bin_size & (bin_size-1) == 0, "Bin size must be a power of two.") + + rank = int(intrinsics.count_trailing_zeros(uint(bin_size)) - ODIN_HEAP_MIN_BIN_SHIFT) + assert_contextless(0 <= rank && rank < ODIN_HEAP_BIN_RANKS, "The heap allocator miscalculated the bin rank; it must be within [0.. int { + assert_contextless(n > 1, "This procedure does not handle the edge case of n < 2.") + return 1 << ((8 /* bits */ * size_of(int)) - intrinsics.count_leading_zeros(uint(n-1))) +} + +/* +Round an arbitrary byte `size` up to a bin size that can fit it. +*/ +@(require_results) +heap_round_to_bin_size :: proc "contextless" (size: int) -> (bin_size: int) { + assert_contextless(0 <= size && size <= ODIN_HEAP_MAX_BIN_SIZE, "Size must be within [0..=ODIN_HEAP_MAX_BIN_SIZE].") + bin_size = round_up_to_power_of_two(max(ODIN_HEAP_MIN_BIN_SIZE, size)) + assert_contextless(bin_size & (bin_size-1) == 0, "The heap allocator miscalculated the bin size; it must be a power of two.") + return +} + +/* +Calculate both the rounded bin size and the rank for an arbitrary byte `size`. +*/ +@(require_results) +heap_calculate_sizes :: proc "contextless" (size: int) -> (bin_size, rank: int) { + bin_size = heap_round_to_bin_size(size) + rank = heap_bin_size_to_rank(bin_size) + return +} + +/* +Find which segment should own an address with bit masking. + +This does not return a valid segment address if the address itself is invalid. +*/ +@(require_results) +find_segment_from_pointer :: #force_inline proc "contextless" (ptr: rawptr) -> ^Heap_Segment { + return cast(^Heap_Segment)(uintptr(ptr) & ~uintptr(heap_get_segment_size()-1)) +} + +/* +Derive the bin index from an address. + +This is used only in debugging. +*/ +@(require_results) +heap_find_bitmapping_from_pointer :: #force_inline proc "contextless" (slab: ^Heap_Slab, ptr: rawptr) -> (sector, index: uint) { + number := uint((uintptr(ptr) - uintptr(slab.data)) / uintptr(slab.bin_size)) + assert_contextless(int(number) < slab.max_bins, "The heap allocator miscalculated the bin number for an address.") + sector = number / (8 /* bits */ * size_of(uint)) + index = number % (8 /* bits */ * size_of(uint)) + return +} + +/* +Atomically push a pointer into `list`'s location and simultaneously move the +old value of `list` to `old_head_destination`. This is used for atomic +linked lists. +*/ +@(private="file") +atomic_pop_push_pointer :: proc "contextless" (list: ^Tagged_Pointer, ptr: rawptr, old_head_destination: ^uintptr) { + old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)list, .Relaxed) + for { + intrinsics.atomic_store_explicit(old_head_destination, cast(uintptr)old_head.pointer, .Release) + new_head := Tagged_Pointer{ + pointer = i64(uintptr(ptr)), + version = old_head.version + 1, + } + + old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)list, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) + if swapped { + break + } + old_head = transmute(Tagged_Pointer)old_head_ + } } // // Data Structures // +// NOTE: No structure with atomic fields in this allocator should ever be +// made `#packed` without regard for alignment to the size of the pointer for +// each atomic field, as misaligned atomic access could cause issues on some +// architectures. + /* -The **Slab** is a fixed-size division of a Superpage, configured at runtime to -contain fixed-size allocations. It uses two bitmaps to keep track of the +The **Slab** is a division of a Segment, configured at runtime to +contain fixed-size allocations. It uses two free lists to keep track of the state of its bins: whether a bin is locally free or remotely free. It is a Slab allocator in its own right, hence the name. -Each allocation is self-aligned, up to an alignment size of `HEAP_MAX_ALIGNMENT` -(64 bytes by default). The slab itself is aligned to its size, allowing -allocations within to find their owning slab in constant time. - -Remote threads, in an atomic wait-free manner, flip bits across `remote_free` -to signal that they have freed memory which does not belong to them. The owning -thread will then collect those bits when the slab is approaching fullness. +Each allocation is self-aligned, up to an alignment size of `ODIN_HEAP_MAX_ALIGNMENT` +(64 bytes by default). +Remote threads, in an atomic lock-free manner, push pointers onto +`remote_free_list` when the Slab is not owned by any heap. Otherwise, remote +frees go directly to the Heap. **Fields**: -`index` is used to make self-referencing easier. It must be the first field in -the Slab, as `heap_slab_clear_data` depends on this. +`data` points to the first bin and is used for calculating bin positions. + + +`prev_slab` and `next_slab` are used when the Slab is added to a linked list. +This can be a linked list of Slabs with the same size or a linked list of free +Slabs on the heap. + + +`free_bins` counts the exact number of free bins known to the allocating +thread. This value does not yet include any remote frees. + +`used_bins` tracks how many unused addresses have been given out, which is used +to find a fresh bin if there are no pointers on the free list. + +`max_bins` is the number of maximum bins that can be allocated from this Slab. +It makes an inclusive range of `0..=n`. + `bin_size` tracks the precise byte size of the allocations. -`is_dirty` indicates that the slab has been used in the past and its memory may -not be entirely zeroed. +`bin_rank` is the cached rank for the `bin_size`, kept for performance purposes. -`is_full` is true at some point after `free_bins` == 0 and false otherwise. -It is atomic and used for inter-thread communication. -`has_remote_frees` is true at some point after a bit has been flipped on in -`remote_free`. It is atomic and used for inter-thread communication. +`capacity` is how much space the Slab was given by the Segment when allocated. -`max_bins` is set at initialization with the number of bins allotted. -`free_bins` counts the exact number of free bins known to the allocating -thread. This value does not yet include any remote free bins. +`double_free_tracker` is a slice that is used only in debug mode. Its raw data +will point to space outside of the segment and contain a bitmap of flags +signalling whether or not a particular bin is free. -`dirty_bins` tracks which bins have been used and must be zeroed before being -handed out again. Because we always allocate linearly, this is simply an -integer indicating the greatest index the slab has ever distributed. +`xor_key` is used only in debug mode to help check for free list corruption. -`next_free_sector` points to which `uint` in `local_free` contains a free bit. -It is always the lowest index possible. -`sectors` is the length of each bitmap. +`free_list` is either nil or points to one of the free bins, which itself may +point to another freed bin, creating a linked list within the Slab space. -`local_free` tracks which bins are free. - -`remote_free` tracks which bins have been freed by other threads. -It is atomic. - -`cached_at` and `cached_index` store where this slab is with regards to the -heap's slab map. - -`data` points to the first bin and is used for calculating bin positions. +`remote_free_list` is an atomic linked list, serving the same role as +`free_list` but for other threads. This allows the allocator to free memory +from other threads in a lock-free manner. */ Heap_Slab :: struct { - index: int, // This field must be the first. - bin_size: int, - is_dirty: bool, + data: uintptr, - max_bins: int, - free_bins: int, - dirty_bins: int, - next_free_sector: int, + prev_slab: ^Heap_Slab, + next_slab: ^Heap_Slab, - is_full: bool, // atomic - remote_free_bins_scheduled: int, // atomic + free_bins: int, + used_bins: int, + max_bins: int, - sectors: int, - local_free: [^]uint, - remote_free: [^]uint, // data referenced is atomic + bin_size: int, + bin_rank: int, - // NOTE: `remote_free` and its contents should be fine with regards to - // alignment for the purposes of atomic access, as every field in a Slab is - // at least register-sized. - // - // Atomically accessing memory that is not aligned to the register size - // may cause an issue on some systems. + capacity: int, - cached_at: ^Heap_Cache_Block, - cached_index: int, + double_free_tracker: []uint, + xor_key: uintptr, - data: uintptr, - - /* ... local_free's data ... */ - /* ... remote_free's data ... */ - /* ... allocation data ... */ + free_list: ^uintptr, + remote_free_list: Tagged_Pointer, // atomic } /* -The **Superpage** is a single allocation from the operating system's virtual -memory subsystem, always with a fixed size at compile time, that is used to -store almost every allocation requested by the program in sub-allocators known -as Slabs. +The **Segment** is a single contiguous allocation from the operating system's +virtual memory subsystem, subdivided into Slabs. All metadata lives at the head +of the allocation. On almost every platform, this structure will be 2MiB by default. Depending on the operating system, addresses within the space occupied by the -superpage (and hence its allocations) may also have faster access times due to +Segment (and hence its allocations) may also have faster access times due to leveraging properties of the Translation Lookaside Buffer. -It is always aligned to its size, allowing any address allocated from its space -to look up the owning superpage in constant-time with bit masking. +It is always aligned to `heap_get_segment_size()`, allowing any address +allocated from its space to look up the Segment in constant-time with bit +masking. **Fields:** -`huge_size` is precisely how many bytes have been allocated to the address at -which this structure resides, if it is a Huge allocation. It is zero for normal -superpages. +`owner` is the value of `get_current_thread_id` for the thread which owns this +Segment or zero if it is orphaned. -`prev` points to the previous superpage in the doubly-linked list of all -superpages for a single thread's heap. +`heap` points to the `Heap` which owns this Segment or is nil if is orphaned. -`next` points to the next superpage in the same doubly-linked list. It is -accessed with atomics only when engaging with the orphanage, as that is the -only time it should change in a parallel situation. For all other cases, the -thread which owns the superpage is the only one to read this field. +`size` is the exact size of the Segment allocation, used when returning the +memory to the operating system. -`remote_free_set` is true if there might be a slab with remote frees in this superpage. -`owner` is the value of `current_thread_id` for the thread which owns this superpage.. +`prev_segment` and `next_segment` are used to add the Segment into linked +lists, whether on a thread's heap or in the orphanage. -`master_cache_block` points to the `local_heap_cache` for the thread which owns this -superpage. This field is used to help synchronize remote freeing. +NOTE: `next_segment` is accessed with atomics only when engaging with the +orphanage, as that is the only time it should change in a parallel situation. +For all other cases, the thread which owns the Segment is the only one to read +this field. -`free_slabs` is the count of slabs which are ready to use for new bin ranks. -`next_free_slab_index` is the lowest index of a free slab. -A value of `HEAP_SLAB_COUNT` means there are no free slabs. +`may_return` is a flag that is set to true after all Slabs have been used +at least once, used as a heuristic to prevent the allocator from freeing the +Segment too early to improve performance. -`cached_at` and `cached_index` store where this superpage is with regards to -the heap's list of superpages with free slabs. -`cache_block` is the space where data for the heap's cache is stored, if this -superpage has been claimed by the heap. It should otherwise be all zero. +`slab_size_class` is the size class of each and every Slab, used for tracking +in what size intervals the Slabs are subdivided. + +`slab_shift` is an unsigned integer that is used to shift an address to a bin, +minus the address to the Segment, to find which Slab the bin is in. + + +`padding` tracks how many bytes were used to get an alignment of +`ODIN_HEAP_MAX_ALIGNMENT` for the first bin. This is used to ensure all bytes +are accounted during the tally of `get_local_heap_info`. + + +`free_slabs` is the count of Slabs which are ready to use for new bin ranks. + +`slabs` is the slice of Slab metadata which contains pointers to each Slab's +starting address and byte capacity. This information is used to subdivide the +Slab when a request is made for a new bin rank. */ -Heap_Superpage :: struct { - huge_size: int, // This field must be the first. - prev: ^Heap_Superpage, - next: ^Heap_Superpage, // atomic in orphanage, otherwise non-atomic +Heap_Segment :: struct { + owner: int, // atomic + heap: ^Heap, // atomic + size: int, - fully_used_once: bool, - has_ever_been_remotely_freed: bool, + prev_segment: ^Heap_Segment, + next_segment: ^Heap_Segment, - remote_free_set: bool, // atomic - owner: int, // atomic - master_cache_block: ^Heap_Cache_Block, // atomic + may_return: bool, + + slab_size_class: Heap_Slab_Class, + slab_shift: uint, + + padding: int, free_slabs: int, - next_free_slab_index: int, - - longest_contiguous_free_slab: int, - contiguous_free_slabs: [HEAP_SLAB_COUNT]int, - - cached_at: ^Heap_Cache_Block, - cached_index: int, - - cache_block: Heap_Cache_Block, + slabs: []Heap_Slab, + /* ... the slab space itself ... */ } /* -`Heap_Cache_Block` is a structure that lives within each `Heap_Superpage` that has been -claimed by a thread's heap to store heap-relevant metadata for improving -allocator performance. This structure makes use of space that would have -otherwise been unused due to the particular alignment needs the allocator has. - -The number of superpages that each `Heap_Cache_Block` oversees is dictated by the -`HEAP_SUPERPAGE_CACHE_RATIO` constant. As a thread's heap accumulates more -superpages, the heap will need to claim additional superpages - at the rate of -that constant - to keep track of all the possible data. - -The arrays are configured in such a manner that they can never overflow, and -the heap will always have more than enough space to record the data needed. - - -The `HEAP_CACHE_SLAB_MAP_STRIDE` is of note, as it must contain the number of slabs -per superpage (31, by default) times the number of superpages overseen per -`Heap_Cache_Block`. This is largely where all the space is allocated, but it pays off -handsomely in allowing the allocator to find if a slab is available for any -particular bin size in constant time. - -In practice, this is an excessive amount of space allocated for a map of arrays -for this purpose, but it is entirely possible that the allocator could get into -a spot where this hash map could overflow if it used any less space. For -instance, if the allocator had as many superpages as possible for one -`Heap_Cache_Block`, filled all the slabs, then the program freed one bin from each -slab, the allocator would need every slot in the map. The cache has as much -space to prevent just that problem from arising. +`Heap` is a thread-local structure that is allocated upon the first allocation +for a thread and stores metadata relevant to the thread's allocator. **Fields:** -`in_use` will be true, if the parent `Heap_Superpage` is using the space occupied by -this struct. This indicates that the superpage should not be freed. - -`next_cache_block` is an atomic pointer to the next `Heap_Cache_Block`. Each -`Heap_Cache_Block` is linked together in this way to allow each heap to expand -its available space for tracking information. +`segments` is a linked list of Segments which belong to this heap. -_The next five fields are only used in the `Heap_Superpage` pointed to by `local_heap`._ +`free_slabs` is an array of linked lists by Slab size class which store slabs +not in use. -`length` is how many `Heap_Cache_Block` structs are in use across the local -thread's heap. - -`owned_superpages` is how many superpages are in use by the local thread's -heap. - -`remote_free_count` is an estimate of how many superpages have slabs with -remote frees available to merge. - -`slab_map_length_by_rank` counts how many entries are in each slab array within -the slab map. - -`superpages_with_free_slabs_length` counts how many entries are in `superpages_with_free_slabs` -across the entire heap cache. +NOTE: The `Heap_Slab_Class.Huge` entry exists for code simplicity. When a +Huge allocation is made, the Slab is immediately taken for the request. Huge +allocations also bypass the orphanage and are returned to the operating system +when freed. -_The next three fields constitute the main data used in this struct, and each -of them are like partitions, spread across a linked list._ +`slabs_by_rank` is an array of linked lists, each list containing Slabs all of +the same size per its rank. For example, the 0th list contains all Slabs that +can fit allocations of `ODIN_HEAP_MIN_BIN_SIZE`. -`slab_map` is a hash map of arrays, keyed by bin rank, used to quickly find a -slab with free bins for a particular bin size. It uses linear probing. -`superpages_with_free_slabs` is an array of superpages with `free_slabs` -greater than zero, used for quickly allocating new slabs when none can be found -in `slab_map`. +`remote_free_list` is an atomic linked list of pointers to bins that have been +freed by other threads which belong to this heap. -`superpages_with_remote_frees` is an atomic set containing references to -superpages that may have slabs with remotely free bins. Superpages are only -added if the slab was full at the time, and an invasive flag keeps the -algorithm from having to search the entire span for an existence check. -Keep in mind that even though a superpage is added to this set when a slab is -full and freed remotely, that state need not persist; the owning thread can -free a bin before the remote frees are merged, invalidating that part of the -heuristic. +`current_memory` reports the amount of memory that the heap has under its control. + +`peak_memory` is the most amount of memory that the heap has ever held. +Both of these values are only updated under debug mode. + + +`prev_heap` and `next_heap` establish the program-wide `global_heap` in debug +mode to detect invalid frees. They are both guarded by `global_heap_lock` and +are not atomic. */ -Heap_Cache_Block :: struct { - in_use: bool, - next_cache_block: ^Heap_Cache_Block, // atomic +Heap :: struct { + segments: ^Heap_Segment, - // { only used in `local_heap` - length: int, - owned_superpages: int, - remote_free_count: int, // atomic + free_slabs: [1+int(max(Heap_Slab_Class))]^Heap_Slab, - slab_map_length_by_rank: [HEAP_BIN_RANKS]int, - superpages_with_free_slabs_length: int, - // } + slabs_by_rank: [ODIN_HEAP_BIN_RANKS]^Heap_Slab, - slab_map: [HEAP_CACHE_SLAB_MAP_STRIDE*HEAP_BIN_RANKS]^Heap_Slab, - superpages_with_free_slabs: [HEAP_SUPERPAGE_CACHE_RATIO]^Heap_Superpage, - superpages_with_remote_frees: [HEAP_SUPERPAGE_CACHE_RATIO]^Heap_Superpage, // atomic + remote_free_list: Tagged_Pointer, // atomic + + current_memory: int, + peak_memory: int, + + prev_heap: ^Heap, + next_heap: ^Heap, } // -// Superstructure Allocation +// Heap Operations +// + +// Push a slab onto a specific linked list. +@(private="file") +_push_slab :: proc "contextless" (list_head: ^^Heap_Slab, slab: ^Heap_Slab) { + slab.prev_slab = nil + slab.next_slab = list_head^ + if list_head^ != nil { + list_head^.prev_slab = slab + } + list_head^ = slab +} + +// Pop a slab off of a specific linked list. +@(private="file") +_pop_slab :: proc "contextless" (list_head: ^^Heap_Slab) -> (slab: ^Heap_Slab) { + assert_contextless(list_head^ != nil, "The heap allocator tried to pop a slab off of an empty list.") + slab = list_head^ + if slab.next_slab != nil { + slab.next_slab.prev_slab = nil + } + list_head^ = slab.next_slab + slab.next_slab = nil + return +} + +// Remove a free slab from the heap, no matter where it is. +heap_remove_free_slab :: proc "contextless" (slab: ^Heap_Slab) { + assert_contextless(slab.bin_size == 0, "The heap allocator tried to remove a slab that is in use from one of the free slab lists.") + for list, index in local_heap.free_slabs { + if list == slab { + local_heap.free_slabs[index] = slab.next_slab + break + } + } + + if slab.prev_slab != nil { + slab.prev_slab.next_slab = slab.next_slab + } + if slab.next_slab != nil { + slab.next_slab.prev_slab = slab.prev_slab + } + slab.prev_slab = nil + slab.next_slab = nil +} + +/* +Add a `slab` that has been configured for allocation to the heap. +*/ +heap_add_ranked_slab :: proc "contextless" (slab: ^Heap_Slab) { + assert_contextless(slab.free_bins > 0, "The heap allocator tried to add a full slab to the ranked lists.") + assert_contextless(slab.bin_size > 0, "The heap allocator tried to add a freed slab to the ranked lists.") + assert_contextless(slab.bin_size <= ODIN_HEAP_MAX_BIN_SIZE, "The heap allocator tried to add a slab configured for a Huge allocation to the ranked lists.") + assert_contextless(slab.bin_rank == heap_bin_size_to_rank(slab.bin_size), "The heap allocator found an incongruent bin rank on a slab.") + rank := slab.bin_rank + + slab.prev_slab = nil + if local_heap.slabs_by_rank[rank] != nil { + local_heap.slabs_by_rank[rank].prev_slab = slab + } + slab.next_slab = local_heap.slabs_by_rank[rank] + local_heap.slabs_by_rank[rank] = slab +} + +/* +Remove a full or free `slab` from the ranked lists. +*/ +heap_remove_ranked_slab :: proc "contextless" (slab: ^Heap_Slab) { + assert_contextless( + slab.max_bins > 0 && ((slab.free_bins == 0) || /* is full */ (slab.free_bins == slab.max_bins)) /* is empty */, + "The heap allocator tried to remove a slab that is not full or not empty from one of the ranked lists.") + assert_contextless(slab.bin_rank == heap_bin_size_to_rank(slab.bin_size), "The heap allocator found an incongruent bin rank on a slab.") + rank := slab.bin_rank + + if slab == local_heap.slabs_by_rank[rank] { + local_heap.slabs_by_rank[rank] = slab.next_slab + } + if slab.prev_slab != nil { + slab.prev_slab.next_slab = slab.next_slab + } + if slab.next_slab != nil { + slab.next_slab.prev_slab = slab.prev_slab + } + slab.prev_slab = nil + slab.next_slab = nil +} + +// +// Allocation // /* -Allocate memory for a Superpage from the operating system and do any initialization work. +Allocate memory for a Segment capable of supporting `bin_size` from the +operating system and do any initialization work. + +An old, empty segment may be passed in `replacement` to convert it to the +requested size class. */ -@(require_results) -heap_make_superpage :: proc "contextless" () -> (superpage: ^Heap_Superpage) { - superpage = cast(^Heap_Superpage)allocate_virtual_memory_superpage() - assert_contextless(uintptr(superpage) & uintptr(SUPERPAGE_SIZE-1) == 0, "The operating system returned virtual memory which isn't aligned to a Superpage-sized boundary.") +heap_make_segment :: proc "contextless" (bin_size: int, replacement: ^Heap_Segment = nil) -> (segment: ^Heap_Segment) { + class := heap_get_size_class(bin_size) - superpage.owner = get_current_thread_id() - superpage.free_slabs = HEAP_SLAB_COUNT + slabs: int + slab_size: int + slab_shift: uint + capacity: int - // Each Slab is aligned to its size, so that finding which slab an - // allocated pointer is assigned to is a constant operation by bit masking. - // - // However, this means we must waste one Slab per Superpage so that the - // Superpage data can live nearby inside the chunk of virtual memory. - base := uintptr(superpage) + HEAP_SLAB_SIZE - for i in 0..= HEAP_MAX_EMPTY_ORPHANED_SUPERPAGES { - intrinsics.atomic_sub_explicit(&heap_orphanage_count, 1, .Relaxed) - free_virtual_memory(superpage, SUPERPAGE_SIZE) - heap_debug_cover(.Superpage_Freed_On_Full_Orphanage) - } else { - heap_push_orphan(superpage) - heap_debug_cover(.Superpage_Pushed_To_Orphanage) + switch class { + case .Small: + slab_shift = intrinsics.constant_log2(ODIN_HEAP_SMALL_SLAB_SIZE) + slab_size = ODIN_HEAP_SMALL_SLAB_SIZE + case .Large, .Huge: + slab_shift = max(uint) + slab_size = capacity } -} + slabs = capacity / slab_size -/* -Remove a superpage from a heap's cache of superpages. -*/ -heap_cache_unregister_superpage :: proc "contextless" (superpage: ^Heap_Superpage) { - if superpage.cache_block.in_use { + if segment == nil { + // The operating system may be out of memory. return } - superpage.fully_used_once = false - local_heap_cache.owned_superpages -= 1 - intrinsics.atomic_store_explicit(&superpage.master_cache_block, nil, .Release) - heap_debug_cover(.Superpage_Unregistered) -} + assert_contextless(uintptr(segment) & uintptr(heap_get_segment_size()-1) == 0, "The operating system returned virtual memory which isn't aligned to the Segment boundary.") + assert_contextless(slabs > 0, "The heap allocator mismanaged the calculation for the number of slabs on making a new segment.") -/* -Add a superpage to a heap's cache of superpages. + // (segment.owner and segment.heap will be set by `heap_add_segment`.) + segment.size = capacity -This will take note if the superpage has free slabs, what are the contents of -its slabs, and it will merge any waiting remote free bins. -*/ -heap_cache_register_superpage :: proc "contextless" (superpage: ^Heap_Superpage) { - // Expand the heap cache's available space if needed. - local_heap_cache.owned_superpages += 1 - if local_heap_cache.owned_superpages / HEAP_SUPERPAGE_CACHE_RATIO > local_heap_cache.length { - tail := local_heap_cache - for /**/; tail.next_cache_block != nil; tail = tail.next_cache_block { } - heap_cache_expand(tail) + segment.slab_size_class = class + segment.slab_shift = slab_shift + + segment.free_slabs = slabs + + // Distribute the allocated space among the substructures. + alloc_at := uintptr(segment) + size_of(Heap_Segment) + + // NOTE: `align_of([]T)` should be 8, so this is safe. + segment.slabs = transmute([]Heap_Slab)Raw_Slice{ + rawptr(alloc_at), + slabs, + } + alloc_at += size_of(Heap_Slab) * uintptr(slabs) + + // Align the pointer to a suitable boundary. + if modulo := alloc_at & (ODIN_HEAP_MAX_ALIGNMENT-1); modulo != 0 { + pad := ODIN_HEAP_MAX_ALIGNMENT - modulo + alloc_at += pad + segment.padding = int(pad) } - // This must come after expansion to prevent a remote thread from running out of space. - // The cache is first expanded, _then_ other threads may know about it. - intrinsics.atomic_store_explicit(&superpage.master_cache_block, local_heap_cache, .Release) + // Carefully setup the first slab, as it has a reduced capacity due to the + // Segment and Slab structures being stored at the start of the segment. + first_slab_capacity := slab_size - int(alloc_at - uintptr(segment)) + assert_contextless(first_slab_capacity >= bin_size, "The heap allocator mismanaged the capacity for the first slab in a new segment.") - // Register the superpage for allocation. - if superpage.free_slabs > 0 { - // Register the superpage as having free slabs available. - heap_cache_add_superpage_with_free_slabs(superpage) - heap_debug_cover(.Superpage_Registered_With_Free_Slabs) + segment.slabs[0].data = alloc_at + segment.slabs[0].capacity = first_slab_capacity + alloc_at += uintptr(first_slab_capacity) + + // The rest of the slabs are full-size. + for &slab in segment.slabs[1:] { + slab.data = alloc_at + slab.capacity = slab_size + alloc_at += uintptr(slab_size) } - // Register slabs. - if superpage.free_slabs < HEAP_SLAB_COUNT { - for i := 0; i < HEAP_SLAB_COUNT; /**/ { - slab := heap_superpage_index_slab(superpage, i) - - if slab.bin_size > HEAP_MAX_BIN_SIZE { - // Skip contiguous slabs. - i += heap_slabs_needed_for_size(slab.bin_size) - } else { - i += 1 - if slab.bin_size == 0 { - continue - } - } - - // When adopting a new Superpage, we take the opportunity to - // merge any remote frees. This is important because it's - // possible for another thread to remotely free memory while - // the thread which owned it is in limbo. - if intrinsics.atomic_load_explicit(&slab.remote_free_bins_scheduled, .Acquire) > 0 { - superpage.has_ever_been_remotely_freed = true - - heap_merge_remote_frees(slab) - - if slab.free_bins > 0 { - // Synchronize with any thread that might be trying to - // free as we merge. - intrinsics.atomic_store_explicit(&slab.is_full, false, .Release) - } - } - - // Free any empty Slabs and register the ones with free bins. - if slab.free_bins == slab.max_bins { - if slab.bin_size > HEAP_MAX_BIN_SIZE { - heap_free_wide_slab(superpage, slab) - } else { - heap_free_slab(superpage, slab) - } - } else if slab.bin_size <= HEAP_MAX_BIN_SIZE && slab.free_bins > 0 { - heap_cache_add_slab(slab, heap_bin_size_to_rank(slab.bin_size)) - } - - heap_debug_cover(.Superpage_Registered_With_Slab_In_Use) - } + // Because the linked lists have stack-like behavior (as opposed to queue), + // we push them in reverse order to better accommodate cache locality of + // contiguous allocations. + list := &local_heap.free_slabs[class] + for i in 1..=slabs { + slab := &segment.slabs[slabs-i] + assert_contextless(slab.data + uintptr(slab.capacity) <= uintptr(segment) + uintptr(capacity), "The heap allocator mismanaged the slab space in a new segment.") + assert_contextless(slab.data % ODIN_HEAP_MAX_ALIGNMENT == 0, "The heap allocator mismanaged the alignment of a slab's first bin in a new segment.") + assert_contextless(find_segment_from_pointer(rawptr(slab.data)) == segment, "The heap allocator was not able to do a reverse lookup of a slab for a new segment.") + assert_contextless(slab.bin_size == 0, "The heap allocator tried to add a non-empty slab to a newly made segment.") + _push_slab(list, slab) } -} -/* -Get a superpage, first by adopting any orphan, or allocating memory from the -operating system if one is not available. -*/ -@(require_results) -heap_get_superpage:: proc "contextless" () -> (superpage: ^Heap_Superpage) { - superpage = heap_pop_orphan() - if superpage == nil { - superpage = heap_make_superpage() - heap_debug_cover(.Superpage_Created_By_Empty_Orphanage) - } else { - heap_debug_cover(.Superpage_Adopted_From_Orphanage) - } - assert_contextless(superpage != nil, "The heap allocator failed to get a superpage.") - return -} - -/* -Make an allocation in the Huge category. -*/ -@(require_results) -heap_make_huge_allocation :: proc "contextless" (size: int) -> (ptr: rawptr) { - // NOTE: ThreadSanitizer may wrongly say that this is the source of a data - // race. This is because a virtual memory address has been allocated once, - // returned to the operating system, then given back to the process in a - // different thread. - // - // It is otherwise impossible for us to race on newly allocated memory. - size := size - if size < SUPERPAGE_SIZE - HEAP_HUGE_ALLOCATION_BOOK_KEEPING { - size = SUPERPAGE_SIZE - heap_debug_cover(.Huge_Alloc_Size_Set_To_Superpage) - } else { - size += HEAP_HUGE_ALLOCATION_BOOK_KEEPING - heap_debug_cover(.Huge_Alloc_Size_Adjusted) - } - assert_contextless(size >= SUPERPAGE_SIZE, "Calculated incorrect Huge allocation size.") - - // All free operations assume every pointer has a Superpage at the - // Superpage boundary of the pointer, and a Huge allocation is no - // different. - // - // The size of the allocation is written as an integer at the beginning, - // but all other fields are left zero-initialized. We then align forward to - // `HEAP_MAX_ALIGNMENT` (64 bytes, by default) and give that to the user. - superpage := cast(^Heap_Superpage)allocate_virtual_memory_aligned(size, SUPERPAGE_SIZE) - assert_contextless(uintptr(superpage) & uintptr(SUPERPAGE_SIZE-1) == 0, "The operating system returned virtual memory which isn't aligned to a Superpage-sized boundary.") - - superpage.huge_size = size - - u := uintptr(superpage) + HEAP_HUGE_ALLOCATION_BOOK_KEEPING - ptr = rawptr(u - u & (HEAP_MAX_ALIGNMENT-1)) - - assert_contextless(uintptr(ptr) & (HEAP_MAX_ALIGNMENT-1) == 0, "Huge allocation is not aligned to HEAP_MAX_ALIGNMENT.") - assert_contextless(find_superpage_from_pointer(ptr) == superpage, "Huge allocation reverse lookup failed.") + heap_add_segment(segment) return } /* -Make an allocation that is at least one entire Slab wide from the provided superpage. - -This will return false if the Superpage lacks enough contiguous Slabs to fit the size. +Configure a Slab that can support an allocation of `bin_size`. */ -@(require_results) -heap_make_slab_sized_allocation :: proc "contextless" (superpage: ^Heap_Superpage, size: int) -> (ptr: rawptr) { - assert_contextless(0 <= superpage.next_free_slab_index && superpage.next_free_slab_index < HEAP_SLAB_COUNT, "Invalid next_free_slab_index.") - contiguous := heap_slabs_needed_for_size(size) +heap_make_slab :: proc "contextless" (bin_size: int) -> (slab: ^Heap_Slab) { + // Get a slab that can fulfill the size request. + class := heap_get_size_class(bin_size) + list := &local_heap.free_slabs[class] - for start := superpage.next_free_slab_index; start < HEAP_SLAB_COUNT-contiguous+1; /**/ { - if superpage.contiguous_free_slabs[start] < contiguous { - // Because this array stores the number of contiguous free slabs, - // we can make good use of that number to jump ahead to the next - // run of free slabs. - start += superpage.contiguous_free_slabs[start] + 1 - continue + // Try to adopt an orphaned segment for the size request. + // + // NOTE: We may end up adopting an in-use segment that cannot fulfill our + // size request. This is acceptable behavior as it will help keep the + // overall memory usage of the program down by redistributing unowned + // segments to heaps that can manage their remote frees. + for { + if heap_adopt_orphan(bin_size, class) == nil { + break } - - // Setup the Slab header. - // This will be a single-sector Slab that may span several Slabs. - slab := heap_superpage_index_slab(superpage, start) - - // Setup slab. - if slab.is_dirty { - heap_slab_clear_data(slab) + if list^ != nil { + break } - slab.bin_size = size - slab.is_full = true - slab.max_bins = 1 - slab.dirty_bins = 1 - slab.sectors = 1 - slab.local_free = cast([^]uint)(uintptr(slab) + size_of(Heap_Slab)) - slab.remote_free = cast([^]uint)(uintptr(slab) + size_of(Heap_Slab) + 1 * size_of(uint)) - data := uintptr(slab) + HEAP_SLAB_ALLOCATION_BOOK_KEEPING - ptr = rawptr(data - data & (HEAP_MAX_ALIGNMENT-1)) - slab.data = uintptr(ptr) - assert_contextless(uintptr(ptr) & (HEAP_MAX_ALIGNMENT-1) == 0, "Slab-wide allocation's data pointer is not correctly aligned.") - assert_contextless(int(uintptr(ptr) - uintptr(superpage)) + size < SUPERPAGE_SIZE, "Incorrectly calculated Slab-wide allocation exceeds Superpage end boundary.") - - // Wipe any non-zero data from slabs ahead of the header. - for x in start+1..= 0, "The heap allocator caused a superpage's free_slabs to go negative.") - if superpage.free_slabs == 0 { - superpage.fully_used_once = true - heap_cache_remove_superpage_with_free_slabs(superpage) - } - - // Cascade contiguous free slab count backwards. - for i := start + contiguous - 1; i > start; i -= 1 { - // Clear out the spots this slab will hold. - superpage.contiguous_free_slabs[i] = 0 - } - j := 0 - for i := start; i >= 0; i -= 1 { - // Rewrite the count behind this slab until it hits a slab in use. - if superpage.contiguous_free_slabs[i] == 0 { - break - } - superpage.contiguous_free_slabs[i] = j - j += 1 - } - heap_update_longest_contiguous_free_slab(superpage) - - // NOTE: Start from zero again, because we may have skipped a non-contiguous block. - heap_update_next_free_slab_index(superpage, 0) - return ptr } - panic_contextless("The heap allocator failed to find a contiguous run of slabs when one of a sufficient length was cached.") -} + if list^ == nil { + // Adoption didn't work. Let's try allocating a new segment. + if heap_make_segment(bin_size) == nil { + // The operating system may be out of memory. + return + } + } -// -// Slabs -// + // At this point, the slab is on the proper list. + slab = _pop_slab(list) -/* -Do everything that is needed to ready a Slab for a specific bin size. + segment := find_segment_from_pointer(slab) + assert_contextless(segment.free_slabs > 0, "The heap allocator was given a slab that belongs to a segment with no free slabs upon trying to make a new slab.") -This involves a handful of space calculations and writing the header. -*/ -@(require_results) -heap_slab_setup :: proc "contextless" (superpage: ^Heap_Superpage, rounded_size: int) -> (slab: ^Heap_Slab) { - assert_contextless(0 <= superpage.next_free_slab_index && superpage.next_free_slab_index < HEAP_SLAB_COUNT, "The heap allocator found a Superpage with an invalid next_free_slab_index.") - assert_contextless(superpage.free_slabs > 0, "The heap allocator tried to setup a Slab in an exhausted Superpage.") + segment.free_slabs -= 1 + if segment.free_slabs == 0 { + // Here we set the flag for the heuristic that helps with freeing + // segments only when they've been thoroughly used. + segment.may_return = true + } - superpage.free_slabs -= 1 - assert_contextless(superpage.free_slabs >= 0, "The heap allocator caused a Superpage's free_slabs to go negative.") + // Set up the slab. + slab.bin_size = bin_size - slab = heap_superpage_index_slab(superpage, superpage.next_free_slab_index) - slab.bin_size = rounded_size + bins := slab.capacity / bin_size + assert_contextless(bins > 0, "The heap allocator miscalculated the number of bins for a new slab.") - // The book-keeping structures compete for the same space as the data, - // so we have to go back and forth with the math a bit. - bins := HEAP_SLAB_SIZE / rounded_size - sectors := bins / INTEGER_BITS - sectors += 0 if bins % INTEGER_BITS == 0 else 1 - - // We'll waste `2 * HEAP_MAX_ALIGNMENT` bytes per slab to simplify the math - // behind getting the pointer to the first bin to align properly. - // Otherwise we'd have to go back and forth even more. - bookkeeping_bin_cost := max(1, int(size_of(Heap_Slab) + HEAP_SECTOR_TYPES * uintptr(sectors) * size_of(uint) + 2 * HEAP_MAX_ALIGNMENT) / rounded_size) - bins -= bookkeeping_bin_cost - sectors = bins / INTEGER_BITS - sectors += 0 if bins % INTEGER_BITS == 0 else 1 - - slab.sectors = sectors slab.free_bins = bins slab.max_bins = bins - if slab.is_dirty { - // Clear only the needed fields. - slab.dirty_bins = bins - slab.next_free_sector = 0 - slab.is_full = false - slab.remote_free_bins_scheduled = 0 - slab.cached_at = nil + + // Detect if this slab was used previously. + if slab.used_bins > 0 { + // Tidy up the slab for re-use. + slab.used_bins = 0 + slab.free_list = nil + intrinsics.mem_zero_volatile(rawptr(slab.data), slab.bin_size * slab.max_bins) } - base_alignment := uintptr(min(HEAP_MAX_ALIGNMENT, rounded_size)) + // We're taking control of this slab, so any remote frees will be + // redirected to our heap instead. + close_free_list(&slab.remote_free_list) - slab_bitmap_base := uintptr(slab) + size_of(Heap_Slab) - total_byte_size_of_all_bitmaps := HEAP_SECTOR_TYPES * uintptr(sectors) * size_of(uint) - pointer_padding := (uintptr(base_alignment) - (slab_bitmap_base + total_byte_size_of_all_bitmaps)) & uintptr(base_alignment - 1) + // (slab.data is already set by `heap_make_segment`.) - // These bitmaps are placed at the end of the struct, one after the other. - slab.local_free = cast([^]uint)(slab_bitmap_base) - slab.remote_free = cast([^]uint)(slab_bitmap_base + uintptr(sectors) * size_of(uint)) - // This pointer is specifically aligned. - slab.data = (slab_bitmap_base + total_byte_size_of_all_bitmaps + pointer_padding) + // Huge allocations are not put into any of the ranked lists. + if class < .Huge { + slab.bin_rank = heap_bin_size_to_rank(bin_size) + heap_add_ranked_slab(slab) + } else { + slab.bin_rank = max(int) + } - assert_contextless(slab.data & (base_alignment-1) == 0, "Incorrect calculation for aligning Slab data pointer.") - assert_contextless(size_of(Heap_Slab) + int(total_byte_size_of_all_bitmaps) + bins * rounded_size < HEAP_SLAB_SIZE, "Slab internal allocation overlimit.") - assert_contextless(find_slab_from_pointer(rawptr(slab.data)) == slab, "Slab data pointer cannot be traced back to its Slab.") - - // Set all of the local free bits. - { - full_sectors := bins / INTEGER_BITS - partial_sector := bins % INTEGER_BITS - for i in 0..= .Double_Free { + // Setup the double free tracker. + // + // NOTE: This will use newly allocated memory outside of the scope of + // the heap which is freed when the slab is. This is acceptable for + // debug mode. + length := bins / (8 /* bits */ * size_of(uint)) + if bins % (8 /* bits */ * size_of(uint)) != 0 { + length += 1 } - if partial_sector > 0 { - slab.local_free[sectors-1] = (1 << uint(bins % INTEGER_BITS)) - 1 - heap_debug_cover(.Slab_Adjusted_For_Partial_Sector) - } - if slab.is_dirty { - for i in 0.. 0 { - slab.remote_free[sectors-1] = 0 - } + slab.double_free_tracker = transmute([]uint)Raw_Slice{ + allocate_virtual_memory(length * size_of(uint)), + length, } } - // Cascade contiguous free slab count backwards. - for i, j := superpage.next_free_slab_index, 0; i >= 0; i -= 1 { - // Rewrite the count behind this slab until it hits a slab in use. - if superpage.contiguous_free_slabs[i] == 0 { - break - } - superpage.contiguous_free_slabs[i] = j - j += 1 - } - heap_update_longest_contiguous_free_slab(superpage) - - // Update the next free slab. - heap_update_next_free_slab_index(superpage, superpage.next_free_slab_index + 1) - - // Make the slab known to the heap. - heap_cache_add_slab(slab, heap_bin_size_to_rank(rounded_size)) - - if superpage.free_slabs == 0 { - superpage.fully_used_once = true - heap_cache_remove_superpage_with_free_slabs(superpage) - heap_debug_cover(.Superpage_Removed_From_Open_Cache_By_Slab) + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + slab.xor_key = uintptr(intrinsics.read_cycle_counter()) * 66_600_049 } return } /* -Set everything after the index field to zero in a Slab. +Get a slab that can fulfill the `size` request. */ -heap_slab_clear_data :: proc "contextless" (slab: ^Heap_Slab) { - intrinsics.mem_zero_volatile(rawptr(uintptr(slab) + size_of(int)), HEAP_SLAB_SIZE - size_of(int)) -} +heap_get_slab :: proc "contextless" (size: int) -> (slab: ^Heap_Slab) { + if size <= ODIN_HEAP_MAX_BIN_SIZE { + bin_size, rank := heap_calculate_sizes(size) -/* -Mark a Slab-wide allocation as no longer in use by the Superpage. -*/ -heap_free_wide_slab :: proc "contextless" (superpage: ^Heap_Superpage, slab: ^Heap_Slab) { - assert_contextless(slab.bin_size > HEAP_MAX_BIN_SIZE, "The heap allocator tried to wide-free a non-wide slab.") - // There is only one bit for a Slab-wide allocation, so this will be easy. - contiguous := heap_slabs_needed_for_size(slab.bin_size) - if superpage.free_slabs == 0 { - heap_cache_add_superpage_with_free_slabs(superpage) - heap_debug_cover(.Superpage_Added_To_Open_Cache_By_Freeing_Wide_Slab) - } - - for i in slab.index..= slab.index; i -= 1 { - // Overwrite the spots this wide slab held. - superpage.contiguous_free_slabs[i] = j - j += 1 - } - for i := slab.index - 1; i >= 0; i -= 1 { - // Expand behind the start until a break is found. - if superpage.contiguous_free_slabs[i] == 0 { - break + slab = local_heap.slabs_by_rank[rank] + if slab == nil { + // The head of the list for this rank is empty, so we'll need to + // make a new one. + slab = heap_make_slab(bin_size) } - superpage.contiguous_free_slabs[i] = j - j += 1 - } - heap_update_longest_contiguous_free_slab(superpage) -} - -/* -Mark a Slab as no longer in use by the Superpage. -*/ -heap_free_slab :: proc "contextless" (superpage: ^Heap_Superpage, slab: ^Heap_Slab) { - if superpage.free_slabs == 0 { - heap_cache_add_superpage_with_free_slabs(superpage) - heap_debug_cover(.Superpage_Added_To_Open_Cache_By_Slab) - } - slab.is_dirty = true - slab.bin_size = 0 - superpage.free_slabs += 1 - assert_contextless(superpage.free_slabs <= HEAP_SLAB_COUNT) - superpage.next_free_slab_index = min(superpage.next_free_slab_index, slab.index) - - // Cascade contiguous free slab count backwards. - j := 1 - if slab.index + 1 < HEAP_SLAB_COUNT { - j += superpage.contiguous_free_slabs[slab.index + 1] - } - superpage.contiguous_free_slabs[slab.index] = j - for i := slab.index - 1; i >= 0; i -= 1 { - if superpage.contiguous_free_slabs[i] == 0 { - break - } - j += 1 - superpage.contiguous_free_slabs[i] = j - } - heap_update_longest_contiguous_free_slab(superpage) -} - -/* -If the Superpage has no slabs in use and is not used by the local heap's cache, -free it from the heap and return true. -*/ -heap_free_superpage_if_empty_and_unused :: proc "contextless" (superpage: ^Heap_Superpage) -> (freed: bool) { - if superpage.free_slabs == HEAP_SLAB_COUNT && superpage.fully_used_once && !superpage.cache_block.in_use { - heap_free_superpage(superpage) - freed = true + assert_contextless(slab.bin_size == heap_round_to_bin_size(size), "The heap allocator found a slab with the wrong bin size during allocation.") + } else { + // We only round the size request for allocations that will fit into Small + // or Large Slabs. For Huge Slabs, their allocations are specifically sized. + slab = heap_make_slab(size) + assert_contextless(slab.bin_size == size, "The heap allocator made a slab with the wrong bin size during allocation.") } return } /* -Merge bins marked as free by remote threads. - -During normal operation, this is only called: - -1. when a slab is about to become full during allocation and there are known - remote frees. -2. when a slab is known to be full and a remote thread has freed a bin. -3. when a superpage is adopted and slabs within are known to have remote frees. -4. when a superpage is being released to the orphanage after clean exit of a - thread, and one of its slabs is known to have remote frees. - -This procedure returns an estimation of the number of remote free bins left to -merge which were not captured in this merge. +Make a new bin-sized allocation, optionally zeroing the memory. */ -heap_merge_remote_frees :: proc "contextless" (slab: ^Heap_Slab) -> (bins_left: int) { - assert_contextless(slab.bin_size > 0, "The heap allocator tried to merge remote frees on an unused slab.") +heap_make_bin :: proc "contextless" (size: int, zero_memory: bool) -> (ptr: rawptr) { + // Get a slab that can fulfill the size request. + slab := heap_get_slab(size) - merged_bins := 0 - next_free_sector := slab.next_free_sector - - // Atomically merge in all of the bits set by other threads. - for i in 0.. 0 { - assert_contextless(heap_superpage_index_slab(superpage, i).bin_size == 0) - superpage.next_free_slab_index = i - heap_debug_cover(.Superpage_Updated_Next_Free_Slab_Index) - return - } else { - i += 1 - } - } - - panic_contextless("The heap allocator was unable to find a free slab in a superpage with free_slabs > 0.") -} - -heap_update_longest_contiguous_free_slab :: proc "contextless" (superpage: ^Heap_Superpage) { - longest := 0 - for i := 0; i < HEAP_SLAB_COUNT; /**/ { - run := superpage.contiguous_free_slabs[i] - when !ODIN_DISABLE_ASSERT { - if run > 0 { - assert_contextless(heap_superpage_index_slab(superpage, i).bin_size == 0) - } - } - longest = max(run, longest) - i += run + 1 - } - superpage.longest_contiguous_free_slab = longest -} - -// -// The Heap Cache -// - -/* -Link a new superpage into the heap. -*/ -heap_link_superpage :: proc "contextless" (superpage: ^Heap_Superpage) { - assert_contextless(superpage.prev == nil, "The heap allocator tried to link an already-linked superpage.") - superpage.prev = local_heap_tail - local_heap_tail.next = superpage - local_heap_tail = superpage - heap_debug_cover(.Superpage_Linked) -} - -/* -Unlink a superpage from the heap. - -There must always be at least one superpage in the heap after the first -non-Huge allocation, in order to track heap metadata. -*/ -heap_unlink_superpage :: proc "contextless" (superpage: ^Heap_Superpage) { - if superpage == local_heap_tail { - assert_contextless(superpage.next == nil, "The heap allocator's tail superpage has a next link.") - assert_contextless(superpage.prev != nil, "The heap allocator's tail superpage has no previous link.") - local_heap_tail = superpage.prev - // We never unlink all superpages, so no need to check validity here. - superpage.prev.next = nil - heap_debug_cover(.Superpage_Unlinked_Tail) - return - } - if superpage.prev != nil { - superpage.prev.next = superpage.next - } - if superpage.next != nil { - superpage.next.prev = superpage.prev - } - heap_debug_cover(.Superpage_Unlinked_Non_Tail) -} - -/* -Mark a superpage from another thread as having had a bin remotely freed. -*/ -heap_remote_cache_add_remote_free_superpage :: proc "contextless" (superpage: ^Heap_Superpage) { - if intrinsics.atomic_exchange_explicit(&superpage.remote_free_set, true, .Acq_Rel) { - // Already set. - heap_debug_cover(.Superpage_Add_Remote_Free_Guarded_With_Set) - return - } - master := intrinsics.atomic_load_explicit(&superpage.master_cache_block, .Acquire) - if master == nil { - // This superpage is not owned by anyone. - // Its remote frees will be acknowledged in whole when it's adopted. - heap_debug_cover(.Superpage_Add_Remote_Free_Guarded_With_Masterless) - return - } - defer heap_debug_cover(.Superpage_Added_Remote_Free) - - cache := master - for { - for i := 0; i < len(cache.superpages_with_remote_frees); i += 1 { - old, swapped := intrinsics.atomic_compare_exchange_strong_explicit(&cache.superpages_with_remote_frees[i], nil, superpage, .Acq_Rel, .Relaxed) - assert_contextless(old != superpage, "A remote thread found a duplicate of a superpage in a heap's superpages_with_remote_frees.") - if swapped { - intrinsics.atomic_add_explicit(&master.remote_free_count, 1, .Release) - return - } - } - next_cache_block := intrinsics.atomic_load_explicit(&cache.next_cache_block, .Acquire) - assert_contextless(next_cache_block != nil, "A remote thread failed to find free space for a new entry in another heap's superpages_with_remote_frees.") - cache = next_cache_block - } -} - -/* -Claim an additional superpage for the heap cache. -*/ -heap_cache_expand :: proc "contextless" (cache: ^Heap_Cache_Block) { - superpage := find_superpage_from_pointer(cache) - assert_contextless(superpage.next != nil, "The heap allocator tried to expand its cache but has run out of linked superpages.") - new_next_cache_block := &(superpage.next).cache_block - assert_contextless(new_next_cache_block.in_use == false, "The heap allocator tried to expand its cache, but the candidate superpage is already using its cache block.") - new_next_cache_block.in_use = true - intrinsics.atomic_store_explicit(&cache.next_cache_block, new_next_cache_block, .Release) - local_heap_cache.length += 1 - heap_debug_cover(.Heap_Expanded_Cache_Data) -} - -/* -Add a slab to the heap's cache, keyed to the bin size rank of `rank`. -*/ -heap_cache_add_slab :: proc "contextless" (slab: ^Heap_Slab, rank: int) { - assert_contextless(slab != nil) - assert_contextless(slab.cached_at == nil) - assert_contextless(slab.bin_size == 1 << (HEAP_MIN_BIN_SHIFT + uint(rank))) - - cache := local_heap_cache - - // Go to the tail of the cache for this rank. - m := local_heap_cache.slab_map_length_by_rank[rank] - for /**/ ; m >= HEAP_CACHE_SLAB_MAP_STRIDE; m -= HEAP_CACHE_SLAB_MAP_STRIDE { - cache = cache.next_cache_block - } - index := rank * HEAP_CACHE_SLAB_MAP_STRIDE + m - - assert_contextless(cache.slab_map[index] == nil) - cache.slab_map[index] = slab - slab.cached_at = cache - slab.cached_index = index - local_heap_cache.slab_map_length_by_rank[rank] += 1 -} - -/* -Get a slab from the heap's cache for an allocation of `rounded_size` bytes. - -If no slabs are already available for the bin rank, one will be created. -*/ -@(require_results) -heap_cache_get_slab :: proc "contextless" (rounded_size: int) -> (slab: ^Heap_Slab) { - rank := heap_bin_size_to_rank(rounded_size) - slab = local_heap_cache.slab_map[rank * HEAP_CACHE_SLAB_MAP_STRIDE] if slab == nil { - superpage := local_heap_cache.superpages_with_free_slabs[0] - if superpage == nil { - superpage = heap_get_superpage() - heap_link_superpage(superpage) - heap_cache_register_superpage(superpage) - } - assert_contextless(superpage.free_slabs > 0) - slab = heap_slab_setup(superpage, rounded_size) + // The operating system may be out of memory. + return } + assert_contextless(slab.free_bins > 0, "The heap allocator was given a slab that had no free bins for an allocation request.") + + if slab.free_list == nil { + assert_contextless(slab.used_bins <= slab.max_bins, "The heap allocator has exceeded the amount of used bins on one of its slabs.") + + // Fetch a new address. + ptr = rawptr(slab.data + uintptr(slab.used_bins * slab.bin_size)) + slab.used_bins += 1 + + when ODIN_HEAP_DEBUG_LEVEL >= .Ensure_Zero { + bytes := cast([^]u8)ptr + for i := 0; i < slab.bin_size; i += 1 { + ensure_contextless(bytes[i] == 0, "The heap allocator's allocation space has been corrupted.") + } + } + } else { + // Pop the pointer off the free list. + ptr = slab.free_list + + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + // Decode the pointer using the slab's key. + ptr = rawptr(uintptr(ptr) ~ slab.xor_key) + + // Derive the bin index. + sector, index := heap_find_bitmapping_from_pointer(slab, ptr) + + // Ensure this address is actually free. + ensure_contextless(slab.data <= uintptr(ptr) && uintptr(ptr) < slab.data + uintptr(slab.capacity), "The heap allocator has detected free list corruption with an address outside of the slab space.") + ensure_contextless(slab.double_free_tracker[sector] & (1 << index) != 0, "The heap allocator has detected free list corruption.") + } + + slab.free_list = (cast(^^uintptr)ptr)^ + + if zero_memory { + // Ensure that the memory zeroing is not optimized out by the compiler. + intrinsics.mem_zero_volatile(ptr, size) + // NOTE: A full memory fence should not be needed for any newly-zeroed + // allocation, as each thread controls its own heap, and for one thread + // to pass a memory address to another implies some secondary + // synchronization method, such as a mutex, which would be the way by + // which the threads come to agree on the state of main memory. + } + } + + slab.free_bins -= 1 + if slab.free_bins == 0 { + // The slab is empty, so it must be taken off the list for its rank to + // prevent further allocation attempts on it. + if slab.bin_size <= ODIN_HEAP_MAX_BIN_SIZE { + // Only allocations that fit into Small and Large Slabs are placed + // into the ranked lists. + heap_remove_ranked_slab(slab) + } + assert_contextless(slab.prev_slab == nil && slab.next_slab == nil, "The heap allocator failed to ensure a full slab was unlinked.") + } + + when ODIN_HEAP_DEBUG_LEVEL >= .Double_Free { + // Derive the bin index. + sector, index := heap_find_bitmapping_from_pointer(slab, ptr) + + // Clear the free bit. + slab.double_free_tracker[sector] &~= 1 << index + } + return } -/* -Remove a slab with the corresponding bin rank from the heap's cache. -*/ -heap_cache_remove_slab :: proc "contextless" (slab: ^Heap_Slab, rank: int) { - assert_contextless(slab != nil) - assert_contextless(slab.cached_at != nil) - assert_contextless(slab.bin_size == 1 << (HEAP_MIN_BIN_SHIFT + uint(rank))) +// +// Remote Freeing +// - source_cache := slab.cached_at - source_index := slab.cached_index - assert_contextless(source_cache.in_use) - assert_contextless(source_cache.slab_map[source_index] == slab) +// NOTE: A remote free list is open when the Slab is not attached to a heap in +// order to receive remote frees in the absence of a heap that can accept them. +// It is otherwise closed. - target_cache := local_heap_cache +@(require_results, private="file") +is_free_list_closed :: #force_inline proc "contextless" (ptr: Tagged_Pointer) -> bool { + return ptr.pointer & HEAP_FREE_LIST_CLOSED == HEAP_FREE_LIST_CLOSED +} - // Get the tail of the slab array for this rank. - m := local_heap_cache.slab_map_length_by_rank[rank] - 1 - assert_contextless(m >= 0) - for /**/ ; m >= HEAP_CACHE_SLAB_MAP_STRIDE; m -= HEAP_CACHE_SLAB_MAP_STRIDE { - target_cache = target_cache.next_cache_block - } - target_index := rank * HEAP_CACHE_SLAB_MAP_STRIDE + m +@(private="file") +close_free_list :: #force_inline proc "contextless" (ptr: ^Tagged_Pointer) { + intrinsics.atomic_or_explicit(cast(^u64)ptr, HEAP_FREE_LIST_CLOSED, .Release) +} - // Swap the slab being removed with the tail. - replacement_slab := target_cache.slab_map[target_index] - assert_contextless(replacement_slab != nil) - - source_cache.slab_map[source_index] = replacement_slab - target_cache.slab_map[target_index] = nil - - replacement_slab.cached_at = source_cache - replacement_slab.cached_index = source_index - - slab.cached_at = nil - slab.cached_index = 0 - - local_heap_cache.slab_map_length_by_rank[rank] -= 1 +@(private="file") +open_free_list :: #force_inline proc "contextless" (ptr: ^Tagged_Pointer) { + intrinsics.atomic_and_explicit(cast(^u64)ptr, ~u64(HEAP_FREE_LIST_CLOSED), .Release) } /* -Make an allocation using contiguous slabs as the backing. - -This procedure will check through the heap's cache for viable superpages to -support the allocation. +Atomically replace a free list's head with nil and return the entire chain. */ @(require_results) -heap_cache_get_contiguous_slabs :: proc "contextless" (size: int) -> (ptr: rawptr) { - contiguous := heap_slabs_needed_for_size(size) - cache := local_heap_cache +heap_take_free_list :: proc "contextless" (list: ^Tagged_Pointer) -> ^uintptr { + old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)list, .Relaxed) for { - for i := 0; i < len(cache.superpages_with_free_slabs); i += 1 { - if cache.superpages_with_free_slabs[i] == nil { - // No superpages with free slabs left. - heap_debug_cover(.Alloc_Slab_Wide_Needed_New_Superpage) - for { - superpage := heap_get_superpage() - heap_link_superpage(superpage) - heap_cache_register_superpage(superpage) - if superpage.longest_contiguous_free_slab >= contiguous { - return heap_make_slab_sized_allocation(superpage, size) - } - } - } else { - heap_debug_cover(.Alloc_Slab_Wide_Used_Available_Superpage) - superpage := cache.superpages_with_free_slabs[i] - if superpage.longest_contiguous_free_slab >= contiguous { - return heap_make_slab_sized_allocation(superpage, size) - } - } + if uintptr(old_head.pointer) & ~uintptr(HEAP_FREE_LIST_CLOSED) == 0 { + // The list is empty. + return nil } - assert_contextless(cache.next_cache_block != nil) - cache = cache.next_cache_block + value := old_head.pointer + new_head := Tagged_Pointer{ + pointer = value & HEAP_FREE_LIST_CLOSED, // Persist the closed state. + version = old_head.version + 1, + } + + old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)list, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) + if swapped { + return cast(^uintptr)rawptr(uintptr(value) & ~uintptr(HEAP_FREE_LIST_CLOSED)) + } + old_head = transmute(Tagged_Pointer)old_head_ + } +} + +// If `list` is open, `ptr` will be pushed onto it. Otherwise, `ptr` will be +// pushed to the remote free list that is on the heap that owns `segment`. +@(private="file") +push_onto_remote_free_list :: proc "contextless" (segment: ^Heap_Segment, list: ^Tagged_Pointer, ptr: rawptr) { + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + ptr := ptr + encoded_ptr := rawptr(uintptr(u64(uintptr(ptr)) ~ global_heap_xor_key)) + } + old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)list, .Relaxed) + for { + if is_free_list_closed(old_head) { + // The list is closed; we must redirect the pointer to the heap. + target_heap := intrinsics.atomic_load_explicit(&segment.heap, .Acquire) + assert_contextless(target_heap != nil, "The heap allocator failed to find the owning heap for a segment which had a closed free list.") + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + atomic_pop_push_pointer(&target_heap.remote_free_list, encoded_ptr, cast(^uintptr)ptr) + } else { + atomic_pop_push_pointer(&target_heap.remote_free_list, ptr, cast(^uintptr)ptr) + } + return + } + + // Write the next address to this pointer, continuing the linked list. + (cast(^uintptr)ptr)^ = uintptr(old_head.pointer) & ~uintptr(HEAP_FREE_LIST_CLOSED) + + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + // Swap to the encoded pointer and push that instead. + ptr = encoded_ptr + } + + new_head := Tagged_Pointer{ + pointer = i64(uintptr(ptr)) | (old_head.pointer & HEAP_FREE_LIST_CLOSED), // Persist the closed state. + version = old_head.version + 1, + } + + old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)list, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) + if swapped { + return + } + old_head = transmute(Tagged_Pointer)old_head_ + } +} + +@(private="file") +merge_slab_remote_free_list :: proc "contextless" (segment: ^Heap_Segment, slab: ^Heap_Slab) { + assert_contextless(slab.bin_size > 0, "The heap allocator tried to merge the remote frees of a slab which is not in use.") + for ptr := heap_take_free_list(&slab.remote_free_list); ptr != nil; /**/ { + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + ptr = cast(^uintptr)(uintptr(u64(uintptr(ptr)) ~ global_heap_xor_key)) + } + next := ptr^ + heap_free_bin(segment, slab, ptr) + ptr = cast(^uintptr)next } } /* -Add a superpage with free slabs to the heap's cache. +Merge any remote frees on the thread's heap. */ -heap_cache_add_superpage_with_free_slabs :: proc "contextless" (superpage: ^Heap_Superpage) { - assert_contextless(intrinsics.atomic_load_explicit(&superpage.owner, .Acquire) == get_current_thread_id(), "The heap allocator tried to cache a superpage that does not belong to it.") - - m := local_heap_cache.superpages_with_free_slabs_length - cache := local_heap_cache - for m >= HEAP_SUPERPAGE_CACHE_RATIO { - m -= HEAP_SUPERPAGE_CACHE_RATIO - cache = cache.next_cache_block +heap_merge_remote_free_list :: proc "contextless" () { + for ptr := heap_take_free_list(&local_heap.remote_free_list); ptr != nil; /**/ { + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + ptr = cast(^uintptr)(uintptr(u64(uintptr(ptr)) ~ global_heap_xor_key)) + } + next := ptr^ + heap_free(ptr) + ptr = cast(^uintptr)next } - assert_contextless(cache.superpages_with_free_slabs[m] == nil) - - cache.superpages_with_free_slabs[m] = superpage - superpage.cached_at = cache - superpage.cached_index = m - local_heap_cache.superpages_with_free_slabs_length += 1 -} - -/* -Remove a superpage from the heap's cache for superpages with free slabs. -*/ -heap_cache_remove_superpage_with_free_slabs :: proc "contextless" (superpage: ^Heap_Superpage) { - m := local_heap_cache.superpages_with_free_slabs_length - 1 - cache := local_heap_cache - for m >= HEAP_SUPERPAGE_CACHE_RATIO { - m -= HEAP_SUPERPAGE_CACHE_RATIO - cache = cache.next_cache_block - } - assert_contextless(cache.superpages_with_free_slabs[m] != nil) - - replacement_superpage := cache.superpages_with_free_slabs[m] - superpage.cached_at.superpages_with_free_slabs[superpage.cached_index] = replacement_superpage - - replacement_superpage.cached_at = superpage.cached_at - replacement_superpage.cached_index = superpage.cached_index - - cache.superpages_with_free_slabs[m] = nil - local_heap_cache.superpages_with_free_slabs_length -= 1 - - superpage.cached_at = nil - superpage.cached_index = 0 } // -// Superpage Orphanage +// Freeing +// + +heap_free_segment :: proc "contextless" (segment: ^Heap_Segment) { + // Remove all slabs belonging to this segment from the heap. + for &slab in segment.slabs { + assert_contextless(slab.bin_size == 0, "The heap allocator found a slab which is not free while freeing a segment.") + heap_remove_free_slab(&slab) + } + + heap_remove_segment(segment) + + // Huge allocations are simply given back to the operating system when done. + // The other segments will be placed into the orphanage if there is room. + if segment.slab_size_class == .Huge || !heap_orphan_empty_segment(segment) { + free_virtual_memory(segment, segment.size) + } +} + +heap_free_slab :: proc "contextless" (segment: ^Heap_Segment, slab: ^Heap_Slab) { + segment.free_slabs += 1 + assert_contextless(segment.free_slabs <= len(segment.slabs), "The heap allocator freed a slab and caused an overflow of the free slab counter.") + + when ODIN_HEAP_DEBUG_LEVEL >= .Double_Free { + // Return the memory specifically allocated for this bitmap back to the operating system. + free_virtual_memory(raw_data(slab.double_free_tracker), len(slab.double_free_tracker) * size_of(uint)) + slab.double_free_tracker = {} + } + + if slab.bin_size <= ODIN_HEAP_MAX_BIN_SIZE { + // Remove the slab from the array of ranked lists so that it is no + // longer used for future allocations. + heap_remove_ranked_slab(slab) + } + + // Mark the slab as free. + slab.bin_size = 0 + + if segment.free_slabs == len(segment.slabs) && segment.may_return { + heap_free_segment(segment) + } else { + // Put the now-freed slab back on the heap. + _push_slab(&local_heap.free_slabs[segment.slab_size_class], slab) + } +} + +heap_free_bin :: proc "contextless" (segment: ^Heap_Segment, slab: ^Heap_Slab, ptr: rawptr) { + slab.free_bins += 1 + + when ODIN_HEAP_DEBUG_LEVEL >= .Double_Free { + // Derive the bin index. + sector, index := heap_find_bitmapping_from_pointer(slab, ptr) + + // Panic if a double free has occurred. + ensure_contextless(slab.double_free_tracker[sector] & (1 << index) == 0, "The heap allocator caught a double free.") + + // Set the free bit. + slab.double_free_tracker[sector] |= 1 << index + } + + assert_contextless(slab.bin_size > 0, "The heap allocator tried to free a pointer belonging to an empty slab.") + assert_contextless(slab.free_bins <= slab.max_bins, "The heap allocator freed a bin and caused an overflow of the free bin counter.") + + // Push onto the head of the free list. + (cast(^^uintptr)ptr)^ = slab.free_list + + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + // Encode the pointer with the heap's key. + xor_key := intrinsics.atomic_load_explicit(&slab.xor_key, .Acquire) + slab.free_list = cast(^uintptr)(uintptr(ptr) ~ xor_key) + } else { + slab.free_list = cast(^uintptr)ptr + } + + if slab.free_bins == 1 && slab.bin_size <= ODIN_HEAP_MAX_BIN_SIZE { + // The slab has free bins again, which means we can place it back + // into its appropriate ranked list. + heap_add_ranked_slab(slab) + } else if slab.free_bins == slab.max_bins && slab.used_bins == slab.max_bins { + heap_free_slab(segment, slab) + } +} + +// +// Segment Orphanage // // This construction is used to avoid the ABA problem. @@ -1284,44 +1188,55 @@ Tagged_Pointer :: bit_field u64 { } /* -Put a Superpage into the global orphanage. - -The caller is responsible for fetch-adding the count. +Push an empty Segment into the global orphanage. */ -heap_push_orphan :: proc "contextless" (superpage: ^Heap_Superpage) { - assert_contextless(intrinsics.atomic_load_explicit(&superpage.owner, .Acquire) != 0, "The heap allocator tried to push an unowned superpage to the orphanage.") +heap_orphan_empty_segment :: proc "contextless" (segment: ^Heap_Segment) -> (accepted: bool) { + when ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS > 0 { + old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&heap_orphanage.empty, .Relaxed) + for { + count := old_head.pointer & ODIN_HEAP_ORPHANAGE_COUNT_BITS + untagged_head := uintptr(old_head.pointer) & ~uintptr(ODIN_HEAP_ORPHANAGE_COUNT_BITS) - // The algorithm below is one of the well-known methods of resolving the - // ABA problem, known as a tagged pointer. The gist is that if another - // thread has changed the value, we'll be able to detect that by checking - // against the version bits. - old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&heap_orphanage, .Relaxed) - intrinsics.atomic_store_explicit(&superpage.master_cache_block, nil, .Seq_Cst) - // NOTE: This next instruction must not float above the previous one, as - // this superpage could host the `master_cache_block`. The order is important - // to keep other threads from trying to access it while we're clearing it. - if superpage.cache_block.in_use { - intrinsics.mem_zero_volatile(&superpage.cache_block, size_of(Heap_Cache_Block)) - heap_debug_cover(.Superpage_Cache_Block_Cleared) + assert_contextless(count <= ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS, "The heap orphanage for empty segments has an invalid embedded `count`.") + if count == ODIN_HEAP_MAX_EMPTY_ORPHANED_SEGMENTS { + break + } + + // Set the next pointer in the list to the current head. + intrinsics.atomic_store_explicit(&segment.next_segment, cast(^Heap_Segment)untagged_head, .Release) + + new_head := Tagged_Pointer{ + pointer = i64(uintptr(segment)) | (count + 1), + version = old_head.version + 1, + } + + old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)&heap_orphanage.empty, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) + if swapped { + accepted = true + break + } + old_head = transmute(Tagged_Pointer)old_head_ + } } - superpage.prev = nil - intrinsics.atomic_store_explicit(&superpage.owner, 0, .Release) - for { - // NOTE: `next` is accessed atomically when pushing or popping from the - // orphanage, because this field must synchronize with other threads at - // this point. - // - // This has to do mainly with swinging the head's linking pointer. - // - // Beyond this point, the thread which owns the superpage will be the - // only one to read `next`, hence why it is not read atomically - // anywhere else. - intrinsics.atomic_store_explicit(&superpage.next, cast(^Heap_Superpage)uintptr(old_head.pointer), .Release) - new_head: Tagged_Pointer = --- - new_head.pointer = i64(uintptr(superpage)) - new_head.version = old_head.version + 1 + return +} - old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)&heap_orphanage, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) +/* +Push a non-empty Segment into the global orphanage. +*/ +heap_orphan_segment :: proc "contextless" (segment: ^Heap_Segment) { + intrinsics.atomic_store_explicit(&segment.owner, 0, .Release) + old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&heap_orphanage.in_use, .Relaxed) + for { + // Set the next pointer in the list to the current head. + intrinsics.atomic_store_explicit(&segment.next_segment, cast(^Heap_Segment)uintptr(old_head.pointer), .Release) + + new_head := Tagged_Pointer{ + pointer = i64(uintptr(segment)), + version = old_head.version + 1, + } + + old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)&heap_orphanage.in_use, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) if swapped { break } @@ -1330,28 +1245,163 @@ heap_push_orphan :: proc "contextless" (superpage: ^Heap_Superpage) { } /* -Remove and return the first entry from the Superpage orphanage, which may be nil. +Push `segment` onto the heap's list. +*/ +heap_add_segment :: proc "contextless" (segment: ^Heap_Segment) { + assert_contextless(segment.prev_segment == nil, "The heap allocator tried to add a segment to its heap which has a non-nil `prev_segment`. This indicates a failure to clear this value.") + + when ODIN_HEAP_DEBUG_LEVEL >= .Invalid_Free { + // We must guard the global heap to prevent another thread from + // interacting with `segments` during this time. + guard_global_heap() + } + + intrinsics.atomic_store_explicit(&segment.owner, get_current_thread_id(), .Release) + intrinsics.atomic_store_explicit(&segment.heap, local_heap, .Release) + segment.next_segment = local_heap.segments + if local_heap.segments != nil { + local_heap.segments.prev_segment = segment + } + local_heap.segments = segment + + when ODIN_HEAP_DEBUG_LEVEL >= .Statistics { + local_heap.current_memory += segment.size + local_heap.peak_memory = max(local_heap.peak_memory, local_heap.current_memory) + } +} + +/* +Remove `segment` from the heap. +*/ +heap_remove_segment :: proc "contextless" (segment: ^Heap_Segment) { + when ODIN_HEAP_DEBUG_LEVEL >= .Invalid_Free { + // We must guard the global heap to prevent another thread from + // interacting with `segments` during this time. + guard_global_heap() + } + + intrinsics.atomic_store_explicit(&segment.owner, 0, .Release) + intrinsics.atomic_store_explicit(&segment.heap, nil, .Release) + if segment == local_heap.segments { + local_heap.segments = segment.next_segment + } + if segment.prev_segment != nil { + segment.prev_segment.next_segment = segment.next_segment + } + if segment.next_segment != nil { + segment.next_segment.prev_segment = segment.prev_segment + } + segment.prev_segment = nil + segment.next_segment = nil + + when ODIN_HEAP_DEBUG_LEVEL >= .Statistics { + local_heap.current_memory -= segment.size + } +} + +/* +Take the first segment from the orphanage. + +If the first one available happens to be an in-use segment, the request for +`bin_size` and `class` are likely to not be satisfied. */ @(require_results) -heap_pop_orphan :: proc "contextless" () -> (superpage: ^Heap_Superpage) { - old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&heap_orphanage, .Relaxed) - for { - superpage = cast(^Heap_Superpage)uintptr(old_head.pointer) - if superpage == nil { - return - } - new_head: Tagged_Pointer = --- - new_head.pointer = i64(uintptr(intrinsics.atomic_load_explicit(&superpage.next, .Acquire))) - new_head.version = old_head.version + 1 +heap_adopt_orphan :: proc "contextless" (bin_size: int, class: Heap_Slab_Class) -> (segment: ^Heap_Segment) { + // First try to get an in-use segment. + { + old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&heap_orphanage.in_use, .Relaxed) + for { + segment = cast(^Heap_Segment)uintptr(old_head.pointer) + if segment == nil { + break + } - old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)&heap_orphanage, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) - if swapped { - intrinsics.atomic_store_explicit(&superpage.next, nil, .Release) - intrinsics.atomic_store_explicit(&superpage.owner, get_current_thread_id(), .Release) - intrinsics.atomic_sub_explicit(&heap_orphanage_count, 1, .Release) - break + next := intrinsics.atomic_load_explicit(&segment.next_segment, .Acquire) + new_head := Tagged_Pointer{ + pointer = i64(uintptr(next)), + version = old_head.version + 1, + } + + old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)&heap_orphanage.in_use, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) + if swapped { + intrinsics.atomic_store_explicit(&segment.next_segment, nil, .Release) + break + } + old_head = transmute(Tagged_Pointer)old_head_ + } + if segment != nil { + heap_add_segment(segment) + + // Get the free slab list in advance. + free_slabs_list := &local_heap.free_slabs[segment.slab_size_class] + + // Block the segment from being freed while we iterate over it. + segment.may_return = false + + // Add the slabs in reverse order to improve cache locality. + for i in 1..=len(segment.slabs) { + slab := &segment.slabs[len(segment.slabs)-i] + assert_contextless(slab.bin_size == 0 || !is_free_list_closed(transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&slab.remote_free_list, .Acquire)), + "The heap allocator found a closed free list on a slab that was just adopted.") + if slab.bin_size == 0 { + _push_slab(free_slabs_list, slab) + } else { + close_free_list(&slab.remote_free_list) + if slab.free_bins > 0 { + heap_add_ranked_slab(slab) + } + // This segment may have remote frees from the time when it + // was orphaned. + merge_slab_remote_free_list(segment, slab) + } + } + + segment.may_return = segment.free_slabs == len(segment.slabs) + } + } + // Next try to get an empty segment if that failed. + if segment == nil { + old_head := transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&heap_orphanage.empty, .Relaxed) + for { + count := old_head.pointer & ODIN_HEAP_ORPHANAGE_COUNT_BITS + untagged_head := uintptr(old_head.pointer) & ~uintptr(ODIN_HEAP_ORPHANAGE_COUNT_BITS) + + segment = cast(^Heap_Segment)uintptr(untagged_head) + if segment == nil { + assert_contextless(count == 0, "The heap allocator saw a nil pointer on the orphanage for empty segments but the count was not zero.") + break + } + + next := intrinsics.atomic_load_explicit(&segment.next_segment, .Acquire) + new_head := Tagged_Pointer{ + pointer = i64(uintptr(next)) | (count - 1), + version = old_head.version + 1, + } + + old_head_, swapped := intrinsics.atomic_compare_exchange_weak_explicit(cast(^u64)&heap_orphanage.empty, transmute(u64)old_head, transmute(u64)new_head, .Acq_Rel, .Relaxed) + if swapped { + intrinsics.atomic_store_explicit(&segment.next_segment, nil, .Release) + break + } + old_head = transmute(Tagged_Pointer)old_head_ + } + if segment != nil { + if segment.slab_size_class == class { + // This segment matches our size class request and all of its slabs should be free. + free_slabs_list := &local_heap.free_slabs[segment.slab_size_class] + for i in 1..=len(segment.slabs) { + slab := &segment.slabs[len(segment.slabs)-i] + assert_contextless(slab.bin_size == 0, "The heap allocator found a slab that is not empty after having adopted from the orphanage for empty segments.") + close_free_list(&slab.remote_free_list) + _push_slab(free_slabs_list, slab) + } + heap_add_segment(segment) + } else if class != .Huge { + // Re-make the segment as we need. + // This procedure will add it to the heap, as well as the empty slabs. + heap_make_segment(bin_size, segment) + } } - old_head = transmute(Tagged_Pointer)old_head_ } return } @@ -1360,92 +1410,144 @@ heap_pop_orphan :: proc "contextless" () -> (superpage: ^Heap_Superpage) { // Globals // -@(init, private) -setup_superpage_orphanage :: proc "contextless" () { - when !VIRTUAL_MEMORY_SUPPORTED { - return +when VIRTUAL_MEMORY_SUPPORTED { + // Upon a child thread's clean exit, this procedure will distribute any + // remaining memory to the orphanage. + // + // Note that this will not run for the main thread, as no thread-local + // cleaner procedures do. + @(private="file") + heap_local_cleanup :: proc "odin" () { + if local_heap == nil { + // A thread without a heap could not have caused any dynamic memory + // to be allocated, thus we exit. + return + } + + for segment := local_heap.segments; segment != nil; /**/ { + next_segment := segment.next_segment + + // Open all the remote free lists on every Slab, so that they can + // hold them until the Segment is claimed by another thread. + for &slab in segment.slabs { + assert_contextless(slab.bin_size == 0 || is_free_list_closed(transmute(Tagged_Pointer)intrinsics.atomic_load_explicit(cast(^u64)&slab.remote_free_list, .Acquire)), + "The heap allocator found an open free list on a slab as the heap's thread was exiting.") + open_free_list(&slab.remote_free_list) + } + + heap_orphan_segment(segment) + + segment = next_segment + } + + // Now that all of the slabs have had their remote free lists opened, + // we should receive no more remote frees on our heap. + // + // It's time to merge all of the heap's remote frees, free it, then exit. + heap_merge_remote_free_list() + + when ODIN_HEAP_DEBUG_LEVEL >= .Invalid_Free { + // Remove this heap from the global heap, iff it has no more active + // allocations. Otherwise, we need to keep the metadata around for + // the duration of the program to monitor the valid space. + if local_heap.segments == nil { + guard_global_heap() + + // Remove this heap from the global heap. + if local_heap == global_heap { + global_heap = local_heap.next_heap + } + if local_heap.prev_heap != nil { + local_heap.prev_heap.next_heap = local_heap.next_heap + } + if local_heap.next_heap != nil { + local_heap.next_heap.prev_heap = local_heap.prev_heap + } + + free_virtual_memory(local_heap, size_of(Heap)) + } + } else { + // The heap itself is an allocation brought about by the very first + // allocation in a thread, thus we free it at the thread's exit. + free_virtual_memory(local_heap, size_of(Heap)) + } } - // Upon a thread's clean exit, this procedure will compact its heap and - // distribute the superpages into the orphanage, if it has space. - add_thread_local_cleaner(proc "odin" () { - for superpage := local_heap; superpage != nil; /**/ { - next_superpage := superpage.next - // The following logic is a specialized case of the same found in - // `compact_heap` that ignores the cache since there's no need to - // update it when the thread's heap is being broken down. - for i := 0; i < HEAP_SLAB_COUNT; /**/ { - slab := heap_superpage_index_slab(superpage, i) - - // Clearing the cache fields is not strictly necessary, but it - // is good for debugging. - slab.cached_at = nil - slab.cached_index = 0 - - if slab.bin_size > HEAP_MAX_BIN_SIZE { - // Skip contiguous slabs. - i += heap_slabs_needed_for_size(slab.bin_size) - } else { - i += 1 - if slab.bin_size == 0 { - continue - } - } - - if intrinsics.atomic_load_explicit(&slab.remote_free_bins_scheduled, .Acquire) > 0 { - heap_merge_remote_frees(slab) - - if slab.free_bins > 0 { - // Synchronize with any thread that might be trying to - // free as we merge. - intrinsics.atomic_store_explicit(&slab.is_full, false, .Release) - } - heap_debug_cover(.Orphaned_Superpage_Merged_Remote_Frees) - } - - if slab.free_bins == slab.max_bins { - if slab.bin_size > HEAP_MAX_BIN_SIZE { - heap_free_wide_slab(superpage, slab) - } else { - heap_free_slab(superpage, slab) - } - heap_debug_cover(.Orphaned_Superpage_Freed_Slab) - } - } - - if superpage.free_slabs == HEAP_SLAB_COUNT { - if intrinsics.atomic_add_explicit(&heap_orphanage_count, 1, .Acq_Rel) >= HEAP_MAX_EMPTY_ORPHANED_SUPERPAGES { - intrinsics.atomic_sub_explicit(&heap_orphanage_count, 1, .Relaxed) - - free_virtual_memory(superpage, SUPERPAGE_SIZE) - heap_debug_cover(.Superpage_Freed_By_Exiting_Thread) - } else { - heap_push_orphan(superpage) - heap_debug_cover(.Superpage_Orphaned_By_Exiting_Thread) - } - } else { - intrinsics.atomic_add_explicit(&heap_orphanage_count, 1, .Release) - heap_push_orphan(superpage) - heap_debug_cover(.Superpage_Orphaned_By_Exiting_Thread) - } - - superpage = next_superpage - } - }) + @(init, private="file") + init_orphanage :: proc "contextless" () { + add_thread_local_cleaner(heap_local_cleanup) + } } -// This is a lock-free, intrusively singly-linked list of Superpages that are -// not in any thread's heap. They are free for adoption by other threads -// needing memory. -heap_orphanage: Tagged_Pointer +/* +This is the global heap orphanage where Segments which are no longer in use by +a specific heap are pushed to. The two fields are lock-free linked lists. -// This is an _estimate_ of the number of Superpages in the orphanage. It can -// never be entirely accurate due to the nature of the design. -heap_orphanage_count: int +`in_use` contains Segments that are either partially or fully allocated +and have been orphaned by their owning heaps. -@(thread_local) local_heap: ^Heap_Superpage -@(thread_local) local_heap_tail: ^Heap_Superpage -@(thread_local) local_heap_cache: ^Heap_Cache_Block +`empty` contains Segments that have been entirely freed, kept on hand for quick +adoption by threads needing memory. It is doubly-tagged in that it supports an +embedded count to prevent acquiring too much unused memory from the operating +system. +*/ +heap_orphanage: struct { + in_use: Tagged_Pointer, + empty: Tagged_Pointer, +} + +// This is the Heap for the current thread. +@(thread_local) local_heap: ^Heap + +when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + // Set only once but may be read from any thread. + @(private) + global_heap_xor_key: u64 + + @(init, private="file") + init_heap_global_xor_key :: proc "contextless" () { + // NOTE: This mask takes into account the upper bits that would be used + // for the version on a `Tagged_Pointer`, the sign bit, as well as the + // bit for `HEAP_FREE_LIST_CLOSED`. + // + // This allows the xor key to be used without interfering with the rest + // of the state on the tagged pointer. + MASK :: 0x00FF_FFFF_FFFF_FFFE + global_heap_xor_key = (u64(intrinsics.read_cycle_counter()) * 66_600_049) & MASK + } +} + +when ODIN_HEAP_DEBUG_LEVEL >= .Invalid_Free { + // For detecting invalid frees, we put all of the heaps on a global list + // protected by a mutex for the sake of simplicity. As this is only enabled + // for this specific debug level and higher, we retain our lock-free + // guarantee on the release mode build of the allocator. + global_heap: ^Heap + global_heap_lock: enum u64 { + Unlocked = 0, + Locked = 1, + } + + lock_global_heap :: proc "contextless" () { + // This is a simple spin lock. + for { + _, swapped := intrinsics.atomic_compare_exchange_weak_explicit(&global_heap_lock, .Unlocked, .Locked, .Acq_Rel, .Relaxed) + if swapped { + break + } + intrinsics.cpu_relax() + } + } + unlock_global_heap :: proc "contextless" () { + _, swapped := intrinsics.atomic_compare_exchange_strong_explicit(&global_heap_lock, .Locked, .Unlocked, .Acq_Rel, .Relaxed) + ensure_contextless(swapped, "A thread tried to unlock the global heap, but it was not locked to begin with.") + } + @(deferred_in=unlock_global_heap) + guard_global_heap :: proc "contextless" () -> bool { + lock_global_heap() + return true + } +} // // API @@ -1456,218 +1558,33 @@ Allocate an arbitrary amount of memory from the heap and optionally zero it. */ @(require_results) heap_alloc :: proc "contextless" (size: int, zero_memory: bool = true) -> (ptr: rawptr) { - assert_contextless(size >= 0, "The heap allocator was given a negative size.") - - // Handle Huge allocations. - if size >= HEAP_HUGE_ALLOCATION_THRESHOLD { - heap_debug_cover(.Alloc_Huge) - return heap_make_huge_allocation(size) - } + assert_contextless(size >= 0, "The heap allocator was given a negative size to allocate.") // Initialize the heap if needed. - if local_heap == nil { - local_heap = heap_get_superpage() - local_heap_tail = local_heap - local_heap_cache = &local_heap.cache_block - local_heap_cache.in_use = true - heap_cache_register_superpage(local_heap) - heap_debug_cover(.Alloc_Heap_Initialized) - } - - // Take care of any remote frees. - remote_free_count := intrinsics.atomic_load_explicit(&local_heap_cache.remote_free_count, .Acquire) - if remote_free_count > 0 { - cache := local_heap_cache - removed := 0 - counter := remote_free_count - // Go through all superpages in the cache for superpages with remote - // frees to see if any still have frees needing merged. - merge_loop: for { - consume_loop: for i in 0.. 0 && slab.free_bins == 0 && intrinsics.atomic_load_explicit(&slab.remote_free_bins_scheduled, .Acquire) > 0 { - heap_merge_remote_frees(slab) - if slab.free_bins == 0 { - // No bins were freed at all, which is possible due - // to the parallel nature of this code. The freeing - // thread could have signalled its intent, but we - // merged before it had a chance to flip the - // necessary bit. - should_reschedule = true - break merge_block - } - intrinsics.atomic_store_explicit(&slab.is_full, false, .Release) - - if bin_size > HEAP_MAX_BIN_SIZE { - heap_free_wide_slab(superpage, slab) - if heap_free_superpage_if_empty_and_unused(superpage) { - assert_contextless(should_reschedule == false, "The heap allocator has freed a superpage and intends to reschedule it for remote free collection.") - break slab_loop - } - } else { - if slab.free_bins == slab.max_bins { - heap_free_slab(superpage, slab) - if heap_free_superpage_if_empty_and_unused(superpage) { - assert_contextless(should_reschedule == false, "The heap allocator has freed a superpage and intends to reschedule it for remote free collection.") - break slab_loop - } - } else { - heap_cache_add_slab(slab, heap_bin_size_to_rank(bin_size)) - } - } - } - - if bin_size > HEAP_MAX_BIN_SIZE { - // Skip contiguous slabs. - j += heap_slabs_needed_for_size(bin_size) - } else { - j += 1 - } - } - - if should_reschedule { - // This is the logic found in `heap_remote_cache_add_remote_free_superpage`, simplified. - if !intrinsics.atomic_exchange_explicit(&superpage.remote_free_set, true, .Acq_Rel) { - cache_reschedule := local_heap_cache - reschedule_loop: for { - for j := 0; j < len(cache_reschedule.superpages_with_remote_frees); j += 1 { - old, swapped := intrinsics.atomic_compare_exchange_strong_explicit(&cache_reschedule.superpages_with_remote_frees[j], nil, superpage, .Acq_Rel, .Relaxed) - assert_contextless(old != superpage, "The heap allocator found a duplicate of a superpage in its superpages_with_remote_frees while rescheduling.") - if swapped { - intrinsics.atomic_add_explicit(&local_heap_cache.remote_free_count, 1, .Release) - break reschedule_loop - } - } - next_cache_block := intrinsics.atomic_load_explicit(&cache_reschedule.next_cache_block, .Acquire) - assert_contextless(next_cache_block != nil, "The heap allocator failed to find free space for a new entry in its superpages_with_remote_frees cache.") - cache_reschedule = next_cache_block - } - } - } else { - removed += 1 - } - - counter -= 1 - if counter == 0 { - break merge_loop - } - } - if cache.next_cache_block == nil { - break merge_loop - } - assert_contextless(cache.next_cache_block != nil) - cache = cache.next_cache_block + if intrinsics.expect(local_heap == nil, false) { + local_heap = cast(^Heap)allocate_virtual_memory(size_of(Heap)) + if intrinsics.expect(local_heap == nil, false) { + // The operating system may be out of memory. + return nil } - if removed > 0 { - intrinsics.atomic_sub_explicit(&local_heap_cache.remote_free_count, removed, .Release) + when ODIN_HEAP_DEBUG_LEVEL >= .Invalid_Free { + if guard_global_heap() { + // Add this heap to the global heap. + local_heap.next_heap = global_heap + if global_heap != nil { + global_heap.prev_heap = local_heap + } + global_heap = local_heap + } } - heap_debug_cover(.Alloc_Collected_Remote_Frees) } - // Handle slab-wide allocations. - if size > HEAP_MAX_BIN_SIZE { - heap_debug_cover(.Alloc_Slab_Wide) - return heap_cache_get_contiguous_slabs(size) - } + // See if there are any remote frees needing to be merged. + heap_merge_remote_free_list() // Get a suitable slab from the heap. - rounded_size := heap_round_to_bin_size(size) - slab := heap_cache_get_slab(rounded_size) - assert_contextless(slab.bin_size == rounded_size, "The heap allocator found a slab with the wrong bin size during allocation.") + ptr = heap_make_bin(size, zero_memory) - // Allocate a bin inside the slab. - sector := slab.next_free_sector - sector_bits := slab.local_free[sector] - assert_contextless(sector_bits != 0, "The heap allocator found a slab with a full next_free_sector.") - - // Select the lowest free bit. - index := uintptr(intrinsics.count_trailing_zeros(sector_bits)) - - // Convert the index to a pointer. - ptr = rawptr(slab.data + (uintptr(sector * SECTOR_BITS) + index) * uintptr(rounded_size)) - when !ODIN_DISABLE_ASSERT { - base_alignment := min(HEAP_MAX_ALIGNMENT, uintptr(rounded_size)) - assert_contextless(uintptr(ptr) & uintptr(base_alignment-1) == 0, "A pointer allocated by the heap is not well-aligned.") - } - - // Clear the free bit. - slab.local_free[sector] &~= (1 << index) - - // Zero the memory, if needed. - if zero_memory && index < uintptr(slab.dirty_bins) { - // Ensure that the memory zeroing is not optimized out by the compiler. - intrinsics.mem_zero_volatile(ptr, rounded_size) - // NOTE: A full memory fence should not be needed for any newly-zeroed - // allocation, as each thread controls its own heap, and for one thread - // to pass a memory address to another implies some secondary - // synchronization method, such as a mutex, which would be the way by - // which the threads come to agree on the state of main memory. - heap_debug_cover(.Alloc_Zeroed_Memory) - } - - // Update statistics. - slab.dirty_bins = int(max(slab.dirty_bins, 1 + sector * SECTOR_BITS + int(index))) - slab.free_bins -= 1 - - // Remove the slab from the cache if the slab's full. - // Otherwise, update the next free sector if the sector's full. - if slab.free_bins == 0 { - slab.next_free_sector = slab.sectors - if intrinsics.atomic_load_explicit(&slab.remote_free_bins_scheduled, .Seq_Cst) > 0 { - heap_merge_remote_frees(slab) - if slab.free_bins == 0 { - // We have come before the other thread, and the bit we needed to find was not set. - // Treat it as if it is full anyway. - intrinsics.atomic_store_explicit(&slab.is_full, true, .Release) - heap_cache_remove_slab(slab, heap_bin_size_to_rank(rounded_size)) - } else { - // No cache adjustment happens here; we know it's already in the cache. - } - } else { - intrinsics.atomic_store_explicit(&slab.is_full, true, .Release) - heap_cache_remove_slab(slab, heap_bin_size_to_rank(rounded_size)) - } - } else { - if slab.local_free[sector] == 0 { - sector += 1 - for /**/; sector < slab.sectors; sector += 1 { - if slab.local_free[sector] != 0 { - break - } - } - slab.next_free_sector = sector - } - } - heap_debug_cover(.Alloc_Bin) return } @@ -1675,143 +1592,42 @@ heap_alloc :: proc "contextless" (size: int, zero_memory: bool = true) -> (ptr: Free memory returned by `heap_alloc`. */ heap_free :: proc "contextless" (ptr: rawptr) { + segment := find_segment_from_pointer(ptr) + // Check for nil. - if ptr == nil { + if intrinsics.expect(ptr == nil, false) { return } - superpage := find_superpage_from_pointer(ptr) - - // Check if this is a huge allocation. - if superpage.huge_size > 0 { - // NOTE: If the allocator is passed a pointer that it does not own, - // which is a scenario that it cannot possibly detect (in any - // reasonably performant fashion), then the condition above may result - // in a segmentation violation. - // - // Regardless, the result of passing a pointer to this heap allocator - // which it did not return is undefined behavior. - free_virtual_memory(superpage, superpage.huge_size) - heap_debug_cover(.Freed_Huge_Allocation) - return - } - - // Find which slab this pointer belongs to. - slab := find_slab_from_pointer(ptr) - assert_contextless(slab.bin_size > 0, "The heap allocator tried to free a pointer belonging to an empty slab.") - - // Check if this is a slab-wide allocation. - if slab.bin_size > HEAP_MAX_BIN_SIZE { - if intrinsics.atomic_load_explicit(&superpage.owner, .Acquire) == get_current_thread_id() { - heap_free_wide_slab(superpage, slab) - heap_free_superpage_if_empty_and_unused(superpage) - heap_debug_cover(.Freed_Wide_Slab) - } else { - // Atomically let the owner know there's a free slab. - intrinsics.atomic_add_explicit(&slab.remote_free_bins_scheduled, 1, .Release) - old := intrinsics.atomic_or_explicit(&slab.remote_free[0], 1, .Seq_Cst) - heap_remote_cache_add_remote_free_superpage(superpage) - - when HEAP_PANIC_ON_DOUBLE_FREE { - if old == 1 { - panic_contextless("The heap allocator freed an already-free pointer.") + when ODIN_HEAP_DEBUG_LEVEL >= .Invalid_Free { + // Sweep through every heap and every segment to see if this one is valid. + pass := false + if guard_global_heap() { + heap_sweep: for heap := global_heap; heap != nil; heap = heap.next_heap { + for heap_segment := heap.segments; heap_segment != nil; heap_segment = heap_segment.next_segment { + if segment == heap_segment { + pass = true + break heap_sweep + } } } - heap_debug_cover(.Remotely_Freed_Wide_Slab) } - return + ensure_contextless(pass, "An invalid free has been detected.") } - // Find which sector and bin this pointer refers to. - bin_number := int(uintptr(ptr) - slab.data) / slab.bin_size - sector := bin_number / INTEGER_BITS - index := uint(bin_number) % INTEGER_BITS + slab := &segment.slabs[(uintptr(ptr) - uintptr(segment)) >> segment.slab_shift] - assert_contextless(bin_number < slab.max_bins, "Calculated an incorrect bin number for slab.") - assert_contextless(sector < slab.sectors, "Calculated an incorrect sector for slab.") - - // See if we own the slab or not, then free the pointer. - if intrinsics.atomic_load_explicit(&superpage.owner, .Acquire) == get_current_thread_id() { - when HEAP_PANIC_ON_DOUBLE_FREE { - if slab.local_free[sector] & (1 << index) != 0 { - panic_contextless("The heap allocator freed an already-free pointer.") - } - } - // Mark the bin as free. - slab.local_free[sector] |= 1 << index - if slab.free_bins == 0 { - intrinsics.atomic_store_explicit(&slab.is_full, false, .Release) - // Put this slab back in the available list. - heap_cache_add_slab(slab, heap_bin_size_to_rank(slab.bin_size)) - heap_debug_cover(.Freed_Bin_Reopened_Full_Slab) - } - slab.free_bins += 1 - assert_contextless(slab.free_bins <= slab.max_bins, "A slab of the heap allocator overflowed its free bins.") - // Free the slab if it's empty and was at one point in time completely - // full. This is a heuristic to prevent a single repeated new/free - // operation in an otherwise empty slab from bogging down the - // allocator. - if slab.free_bins == slab.max_bins && slab.dirty_bins == slab.max_bins { - heap_cache_remove_slab(slab, heap_bin_size_to_rank(slab.bin_size)) - heap_free_slab(superpage, slab) - heap_debug_cover(.Freed_Bin_Freed_Slab_Which_Was_Fully_Used) - } else { - slab.next_free_sector = min(slab.next_free_sector, sector) - heap_debug_cover(.Freed_Bin_Updated_Slab_Next_Free_Sector) - } - heap_free_superpage_if_empty_and_unused(superpage) + // Depending on whether or not we own the address space for the pointer, we + // will either free it directly and immediately or push it to a remote free + // list. + // + // This remote free list will be on the heap, if the owner is still active. + // Otherwise, the remote free will be pushed onto the slab's list whereupon + // it will be merged when another thread adopts its segment. + if intrinsics.atomic_load_explicit(&segment.owner, .Acquire) == get_current_thread_id() { + heap_free_bin(segment, slab, ptr) } else { - // Atomically let the owner know there's a free bin. - - // NOTE: The order of operations here is important. - // - // 1. We must first check if the slab is full. If we wait until later, - // we risk causing a race. - is_full := intrinsics.atomic_load_explicit(&slab.is_full, .Acquire) - - // 2. We have to let the owner know that we intend to schedule a remote - // free. This way, it can keep the cached entry around if it isn't - // able to merge all of them due to timing differences on our part. - intrinsics.atomic_add_explicit(&slab.remote_free_bins_scheduled, 1, .Release) - - // (Technically, the compiler or processor is allowed to re-order the - // above two operations, and this is okay. However, the next one acts - // as a full barrier due to its Sequential Consistency ordering.) - - // 3. Finally, we flip the bit of the bin we want freed. This must be - // the final operation across all cores, because the owner could - // already be in the process of merging remote frees, and if our bin - // was the last one, it has the authorization to wipe the slab and - // possibly reallocate in the same block of memory. - // - // By deferring this to the end, we avoid any possibility of a race, - // even if multiple threads are in this section. - old := intrinsics.atomic_or_explicit(&slab.remote_free[sector], 1 << index, .Seq_Cst) - - // 4. Now we can safely check if the slab is full and add it to the - // owner's cache if needed. - if is_full { - heap_remote_cache_add_remote_free_superpage(superpage) - heap_debug_cover(.Remotely_Freed_Bin_Caused_Remote_Superpage_Caching) - } - - when HEAP_PANIC_ON_DOUBLE_FREE { - if old & (1 << index) != 0 { - panic_contextless("The heap allocator freed an already-free pointer.") - } - } - heap_debug_cover(.Remotely_Freed_Bin) - - // NOTE: It is possible for two threads to be here at the same time, - // thus violating any sense of order among the actual number of bins - // free and the reported number. - // - // T1 & T2 could flip bits, - // T2 could increment the free count, then - // T3 could merge the free bins. - // - // This scenario would result in an inconsistent count no matter which - // way the above procedure is carried out, hence its unreliability. + push_onto_remote_free_list(segment, &slab.remote_free_list, ptr) } } @@ -1820,197 +1636,24 @@ Resize memory returned by `heap_alloc`. */ @(require_results) heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int, zero_memory: bool = true) -> (new_ptr: rawptr) { - Size_Category :: enum { - Unknown, - Bin, - Slab, - Huge, + // Handle `nil` as if it was a new allocation. + // This is the behavior seen in C's `realloc`. + if old_ptr == nil { + return heap_alloc(new_size, zero_memory) } - // We need to first determine if we're crossing size categories. - old_category: Size_Category - switch { - case old_size <= HEAP_MAX_BIN_SIZE: old_category = .Bin - case old_size < HEAP_HUGE_ALLOCATION_THRESHOLD: old_category = .Slab - case old_size >= HEAP_HUGE_ALLOCATION_THRESHOLD: old_category = .Huge - case: unreachable() - } - new_category: Size_Category - switch { - case new_size <= HEAP_MAX_BIN_SIZE: new_category = .Bin - case new_size < HEAP_HUGE_ALLOCATION_THRESHOLD: new_category = .Slab - case new_size >= HEAP_HUGE_ALLOCATION_THRESHOLD: new_category = .Huge - case: unreachable() - } - assert_contextless(old_category != .Unknown) - assert_contextless(new_category != .Unknown) + same_rank := false - if new_category != old_category { - // A change in size category cannot be optimized. - new_ptr = heap_alloc(new_size, false) - intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) - if zero_memory && new_size > old_size { - intrinsics.mem_zero_volatile(rawptr(uintptr(new_ptr) + uintptr(old_size)), new_size - old_size) - } - heap_free(old_ptr) - heap_debug_cover(.Resize_Crossed_Size_Categories) - return - } - - // NOTE: Superpage owners are the only ones that can change the amount of - // memory that backs each pointer. Other threads have to allocate and copy. - - // Check if this is a huge allocation. - superpage := find_superpage_from_pointer(old_ptr) - if superpage.huge_size > 0 { - // This block follows the preamble in `heap_make_huge_allocation`. - new_real_size := new_size - if new_size < SUPERPAGE_SIZE - HEAP_HUGE_ALLOCATION_BOOK_KEEPING { - new_real_size = SUPERPAGE_SIZE - heap_debug_cover(.Resize_Huge_Size_Set_To_Superpage) - } else { - new_real_size += HEAP_HUGE_ALLOCATION_BOOK_KEEPING - heap_debug_cover(.Resize_Huge_Size_Adjusted) - } - resized_superpage := cast(^Heap_Superpage)resize_virtual_memory(superpage, superpage.huge_size, new_real_size, SUPERPAGE_SIZE) - assert_contextless(uintptr(resized_superpage) & (SUPERPAGE_SIZE-1) == 0, "After resizing a huge allocation, the pointer was no longer aligned to a superpage boundary.") - resized_superpage.huge_size = new_real_size - u := uintptr(resized_superpage) + HEAP_HUGE_ALLOCATION_BOOK_KEEPING - new_ptr = rawptr(u - u & (HEAP_MAX_ALIGNMENT-1)) - - if zero_memory && new_size > old_size { - intrinsics.mem_zero_volatile( - rawptr(uintptr(new_ptr) + uintptr(old_size)), - new_size - old_size, - ) - heap_debug_cover(.Resize_Huge_Caused_Memory_Zeroing) - } - - heap_debug_cover(.Resize_Huge) - return - } - - // Find which slab this pointer belongs to. - slab := find_slab_from_pointer(old_ptr) - - // Check if this is a slab-wide allocation. - if slab.bin_size > HEAP_MAX_BIN_SIZE { - contiguous_old := heap_slabs_needed_for_size(slab.bin_size) - contiguous_new := heap_slabs_needed_for_size(new_size) - if contiguous_new == contiguous_old { - // We already have enough slabs to serve the request. - if zero_memory && new_size > old_size { - intrinsics.mem_zero_volatile( - rawptr(uintptr(old_ptr) + uintptr(old_size)), - new_size - old_size, - ) - heap_debug_cover(.Resize_Wide_Slab_Caused_Memory_Zeroing) - } - heap_debug_cover(.Resize_Wide_Slab_Kept_Old_Pointer) - return old_ptr - } - - if slab.index + contiguous_new >= HEAP_SLAB_COUNT { - // Expanding this slab would go beyond the Superpage. - // We need more memory. - new_ptr = heap_alloc(new_size, zero_memory) - intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) - heap_free(old_ptr) - return - } - - if intrinsics.atomic_load_explicit(&superpage.owner, .Acquire) != get_current_thread_id() { - // We are not the owner of this data, therefore none of the special - // optimized paths for wide slabs are available to us, as they all - // involve touching the Superpage. - // - // We must re-allocate. - new_ptr = heap_alloc(new_size, zero_memory) - intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) - heap_free(old_ptr) - heap_debug_cover(.Resize_Wide_Slab_From_Remote_Thread) - return - } - - // Can we shrink the wide slab, or can we expand it in-place? - if contiguous_new < contiguous_old { - previously_full_superpage := superpage.free_slabs == 0 - for i := slab.index + contiguous_new; i < slab.index + contiguous_old; i += 1 { - // Mark the latter slabs as unused. - next_slab := heap_superpage_index_slab(superpage, i) - next_slab.index = i - next_slab.is_dirty = true - next_slab.bin_size = 0 - } - superpage.next_free_slab_index = min(superpage.next_free_slab_index, slab.index + contiguous_new) - superpage.free_slabs += contiguous_old - contiguous_new - slab.bin_size = new_size - if previously_full_superpage { - heap_cache_add_superpage_with_free_slabs(superpage) - heap_debug_cover(.Superpage_Added_To_Open_Cache_By_Resizing_Wide_Slab) - } - heap_debug_cover(.Resize_Wide_Slab_Shrunk_In_Place) - - // Cascade contiguous free slab count backwards. - j := 1 - if slab.index + contiguous_old < HEAP_SLAB_COUNT { - j += superpage.contiguous_free_slabs[slab.index + contiguous_old] - } - for i := slab.index + contiguous_old - 1; i >= slab.index + contiguous_new; i -= 1 { - superpage.contiguous_free_slabs[i] = j - j += 1 - } - heap_update_longest_contiguous_free_slab(superpage) - } else { - // NOTE: We've already guarded against going beyond `HEAP_SLAB_COUNT` in the section above. - for i := slab.index + contiguous_old; i < slab.index + contiguous_new; i += 1 { - if heap_superpage_index_slab(superpage, i).bin_size != 0 { - // Contiguous space is unavailable. - new_ptr = heap_alloc(new_size, zero_memory) - intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, old_size) - heap_free(old_ptr) - heap_debug_cover(.Resize_Wide_Slab_Failed_To_Find_Contiguous_Expansion) - return - } - } - for i := slab.index + contiguous_old; i < slab.index + contiguous_new; i += 1 { - // Wipe the index bits, and if needed, the rest of the data. - next_slab := heap_superpage_index_slab(superpage, i) - next_slab.index = 0 - if next_slab.is_dirty { - heap_slab_clear_data(next_slab) - } - } - // We must update the slab's `bin_size` before calling - // `heap_update_next_free_slab`, as that procedure will need to - // iterate over the slabs. - slab.bin_size = new_size - superpage.free_slabs += contiguous_old - contiguous_new - heap_debug_cover(.Resize_Wide_Slab_Expanded_In_Place) - - // Expand contiguous free slab count forwards. - for i := slab.index + contiguous_old; i < slab.index + contiguous_new; i += 1 { - superpage.contiguous_free_slabs[i] = 0 - } - heap_update_longest_contiguous_free_slab(superpage) - - if superpage.free_slabs == 0 { - superpage.fully_used_once = true - heap_cache_remove_superpage_with_free_slabs(superpage) - } else { - heap_update_next_free_slab_index(superpage, 0) - } - } - - // The slab-wide allocation has been resized in-place. - - return old_ptr - } - - // See if a bin rank change is needed. - new_rounded_size := heap_round_to_bin_size(new_size) - if slab.bin_size == new_rounded_size { + if old_size <= ODIN_HEAP_MAX_BIN_SIZE && new_size <= ODIN_HEAP_MAX_BIN_SIZE { + rounded_old_size := heap_round_to_bin_size(old_size) + rounded_new_size := heap_round_to_bin_size(new_size) + same_rank = rounded_old_size == rounded_new_size + } + + if same_rank { + // We can re-use the same bin. if zero_memory && new_size > old_size { + // Zero any old, dirty memory in the expanded region. intrinsics.mem_zero_volatile( rawptr(uintptr(old_ptr) + uintptr(old_size)), new_size - old_size, @@ -2023,15 +1666,21 @@ heap_resize :: proc "contextless" (old_ptr: rawptr, old_size: int, new_size: int // That is to say, if one thread resizes an address in-place, it's // expected that other threads will need to be notified of this by // the program, as with any other synchronization. - heap_debug_cover(.Resize_Caused_Memory_Zeroing) } - heap_debug_cover(.Resize_Kept_Old_Pointer) - return old_ptr + new_ptr = old_ptr + } else { + // A change in bin rank requires a new bin; this allocator does no coalescence. + new_ptr = heap_alloc(new_size, false) + if intrinsics.expect(new_ptr == nil, false) { + // The operating system may be out of memory. + return + } + intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) + if zero_memory && new_size > old_size { + intrinsics.mem_zero_volatile(rawptr(uintptr(new_ptr) + uintptr(old_size)), new_size - old_size) + } + heap_free(old_ptr) } - // Allocate and copy, as a last resort. - new_ptr = heap_alloc(new_size, zero_memory) - intrinsics.mem_copy_non_overlapping(new_ptr, old_ptr, min(old_size, new_size)) - heap_free(old_ptr) return } diff --git a/base/runtime/heap_allocator_info.odin b/base/runtime/heap_allocator_info.odin index 0ba6b0706..eb0e849cc 100644 --- a/base/runtime/heap_allocator_info.odin +++ b/base/runtime/heap_allocator_info.odin @@ -4,169 +4,159 @@ package runtime import "base:intrinsics" +import "base:runtime" /* Heap_Info provides metrics on a single thread's heap memory usage. + +`total_memory_allocated_from_system` + = `total_memory_used_for_book_keeping` + + `total_memory_in_use` + + `total_memory_free` + +NOTE: `total_memory_used_by_huge_segments` highlights how much memory is used +by the Huge class of allocations and is not part of any equation. */ Heap_Info :: struct { total_memory_allocated_from_system: int `fmt:"M"`, total_memory_used_for_book_keeping: int `fmt:"M"`, + total_memory_used_by_huge_segments: int `fmt:"M"`, total_memory_in_use: int `fmt:"M"`, total_memory_free: int `fmt:"M"`, total_memory_dirty: int `fmt:"M"`, - total_memory_remotely_free: int `fmt:"M"`, - total_superpages: int, - total_superpages_dedicated_to_heap_cache: int, - total_huge_allocations: int, - total_slabs: int, - total_slabs_in_use: int, + total_segments: int, + total_small_segments: int, + total_large_segments: int, + total_huge_segments: int, - total_dirty_bins: int, - total_free_bins: int, - total_bins_in_use: int, - total_remote_free_bins: int, + total_heap_remote_frees: int, + total_slabs: int, - heap_slab_map_entries: int, - heap_superpages_with_free_slabs: int, - heap_slabs_with_remote_frees: int, + total_free_slabs_by_class: [1+int(max(Heap_Slab_Class))]int, + + slabs_by_rank: [runtime.ODIN_HEAP_BIN_RANKS]struct { + total_memory_in_use: int `fmt:"M"`, + + total_slabs: int, + total_bins_in_use: int, + total_free_bins: int, + total_dirty_bins: int, + total_bins: int, + }, + + peak_memory: int `fmt:"M"`, } /* Get information about the current thread's heap. - -This will do additional sanity checking on the heap if assertions are enabled. */ @(require_results) get_local_heap_info :: proc "contextless" () -> (info: Heap_Info) { - if local_heap_cache != nil { - cache := local_heap_cache - slab_map_terminated: [HEAP_BIN_RANKS]bool + exists_in_list :: proc "contextless" (list: ^Heap_Slab, value: ^Heap_Slab) -> bool { + for slab := list; slab != nil; slab = slab.next_slab { + if slab == value { + return true + } + } + return false + } - for { - for rank := 0; rank < HEAP_BIN_RANKS; rank += 1 { - for i := 0; i < HEAP_CACHE_SLAB_MAP_STRIDE; i += 1 { - slab := cache.slab_map[rank * HEAP_CACHE_SLAB_MAP_STRIDE + i] - if slab_map_terminated[rank] { - assert_contextless(slab == nil, "The heap allocator has a gap in its slab map.") - } else if slab == nil { - slab_map_terminated[rank] = true - } else { - info.heap_slab_map_entries += 1 - assert_contextless(slab.bin_size != 0, "The heap allocator has an empty slab in its slab map.") - assert_contextless(slab.bin_size == 1 << (HEAP_MIN_BIN_SHIFT + uint(rank)), "The heap allocator has a slab in the wrong sub-array of the slab map.") - } + if local_heap == nil { + return + } + + for ptr := heap_take_free_list(&local_heap.remote_free_list); ptr != nil; /**/ { + when ODIN_HEAP_DEBUG_LEVEL >= .Free_List_Corruption { + ptr = cast(^uintptr)(uintptr(u64(uintptr(ptr)) ~ global_heap_xor_key)) + } + next := ptr^ + info.total_heap_remote_frees += 1 + // Merge the remote frees, as putting them back would be complicated. + heap_free(ptr) + ptr = cast(^uintptr)next + } + + total_slabs_seen: int + + // Get info on the segments. + for segment := local_heap.segments; segment != nil; segment = segment.next_segment { + assert_contextless(intrinsics.atomic_load_explicit(&segment.owner, .Acquire) == get_current_thread_id(), "A segment has been found in this thread's heap that does not belong to it.") + assert_contextless(intrinsics.atomic_load_explicit(&segment.heap, .Acquire) == local_heap, "A segment has been found in this heap that has not been assigned to it.") + + info.total_slabs += len(segment.slabs) + info.total_memory_allocated_from_system += segment.size + info.total_memory_used_for_book_keeping += int(uintptr(segment.slabs[0].data) - uintptr(segment)) + info.total_segments += 1 + + switch segment.slab_size_class { + case .Small: + info.total_small_segments += 1 + case .Large: + info.total_large_segments += 1 + case .Huge: + total_slabs_seen += 1 + info.total_huge_segments += 1 + info.total_memory_in_use += segment.slabs[0].bin_size + info.total_memory_dirty += segment.slabs[0].bin_size + info.total_memory_used_for_book_keeping += ODIN_HEAP_MAX_ALIGNMENT - segment.padding + info.total_memory_used_by_huge_segments += segment.slabs[0].bin_size + } + + // This block is merely for sanity checking. + for &slab in segment.slabs { + if slab.bin_size == 0 { + assert_contextless(exists_in_list(local_heap.free_slabs[segment.slab_size_class], &slab)) + } else { + switch segment.slab_size_class { + case .Small, .Large: + rank := heap_bin_size_to_rank(slab.bin_size) + assert_contextless(exists_in_list(local_heap.slabs_by_rank[rank], &slab)) + case .Huge: + break } } - for superpage in cache.superpages_with_free_slabs { - if superpage != nil { - info.heap_superpages_with_free_slabs += 1 - } - } - for i in 0.. 0 { - info.total_huge_allocations += 1 - info.total_memory_allocated_from_system += superpage.huge_size - info.total_memory_in_use += superpage.huge_size - HEAP_HUGE_ALLOCATION_BOOK_KEEPING - info.total_memory_used_for_book_keeping += HEAP_HUGE_ALLOCATION_BOOK_KEEPING - } else { - if superpage.cache_block.in_use { - info.total_superpages_dedicated_to_heap_cache += 1 - } - info.total_memory_allocated_from_system += SUPERPAGE_SIZE - for i := 0; i < HEAP_SLAB_COUNT; /**/ { - slab := heap_superpage_index_slab(superpage, i) - - if slab.bin_size != 0 { - info.total_slabs_in_use += 1 - info.total_memory_in_use += slab.bin_size * (slab.max_bins - slab.free_bins) - info.total_memory_free += slab.bin_size * slab.free_bins - info.total_memory_dirty += slab.bin_size * slab.dirty_bins - info.total_bins_in_use += slab.max_bins - slab.free_bins - info.total_free_bins += slab.free_bins - info.total_dirty_bins += slab.dirty_bins - assert_contextless(slab.dirty_bins >= slab.max_bins - slab.free_bins, "A slab of the heap allocator has a number of dirty bins which is not equivalent to the number of its total bins minus the number of free bins.") - // Account for the bitmaps used by the Slab. - info.total_memory_used_for_book_keeping += int(slab.data - uintptr(slab)) - // Account for the space not used by the bins or the bitmaps. - n := int(slab.data - uintptr(slab) + uintptr(slab.max_bins * slab.bin_size)) - if slab.bin_size > HEAP_MAX_BIN_SIZE { - info.total_memory_used_for_book_keeping += heap_slabs_needed_for_size(slab.bin_size) * HEAP_SLAB_SIZE - n - } else { - info.total_memory_used_for_book_keeping += HEAP_SLAB_SIZE - n - } - remote_free_bins := 0 - for j in 0.. HEAP_MAX_BIN_SIZE { - // Skip contiguous slabs. - i += heap_slabs_needed_for_size(slab.bin_size) - } else { - i += 1 - } - } - // Every superpage has to sacrifice one Slab's worth of space so - // that they're all aligned. - info.total_memory_used_for_book_keeping += HEAP_SLAB_SIZE - info.total_slabs += HEAP_SLAB_COUNT - } - when !ODIN_DISABLE_ASSERT { - contiguous_counter := superpage.contiguous_free_slabs[0] - total_contiguous_slabs := contiguous_counter - for i in 1.. runtime.HEAP_MAX_BIN_SIZE { - // Skip contiguous slabs. - i += runtime.heap_slabs_needed_for_size(slab.bin_size) - } else { - i += 1 - } - } - log.info("") - superpage = superpage.next - } -} - -validate_cache :: proc() { - cache := runtime.local_heap_cache - slab_map_terminated: [runtime.HEAP_BIN_RANKS]bool - superpages_with_free_slabs_terminated: bool - for { - // Validate the slab map. - for rank in 0.. 0) - } - } - - next_cache := intrinsics.atomic_load_explicit(&cache.next_cache_block, .Acquire) - if next_cache == nil { - break - } - cache = next_cache - } -} - // // Allocation API Testing // @@ -239,7 +163,7 @@ test_alloc_write_free :: proc( for o in 1..=u64(object_count) { seed := u64(intrinsics.read_cycle_counter()) * o - alignment := min(size, runtime.HEAP_MAX_ALIGNMENT) + alignment := min(size, runtime.ODIN_HEAP_MAX_ALIGNMENT) bytes, alloc_err := allocator.procedure(allocator.data, .Alloc, size, alignment, nil, 0) expect(alloc_err == nil) @@ -296,9 +220,9 @@ test_alloc_write_free :: proc( } - if o % max(1, u64(object_count / 20)) == 0 { - validate_cache() - } + // if o % max(1, u64(object_count / 20)) == 0 { + // validate_cache() + // } } if end_index - start_index != 0 || free_strategy == .At_The_End { @@ -319,7 +243,7 @@ test_continuous_allocation_of_size_n :: proc(count: int, max_size: int) { allocator := context.allocator base_seed := u64(intrinsics.read_cycle_counter()) for size in 0.. 0 && size % (runtime.HEAP_MAX_BIN_SIZE/8) == 0 { + if size > 0 && size % (runtime.ODIN_HEAP_MAX_BIN_SIZE/8) == 0 { log.infof("... %i ...", size) } - alignment := min(size, runtime.HEAP_MAX_ALIGNMENT) + alignment := min(size, runtime.ODIN_HEAP_MAX_ALIGNMENT) // Allocate and free twice to make sure that the memory is truly zeroed. // @@ -393,7 +317,7 @@ test_single_alloc_and_resize :: proc(start, target: int) { allocator := context.allocator base_seed := u64(intrinsics.read_cycle_counter()) - alignment := min(start, runtime.HEAP_MAX_ALIGNMENT) + alignment := min(start, runtime.ODIN_HEAP_MAX_ALIGNMENT) seed := base_seed * (1+u64(start)) bytes, alloc_err := allocator.procedure(allocator.data, .Alloc, start, alignment, nil, 0) @@ -408,6 +332,7 @@ test_single_alloc_and_resize :: proc(start, target: int) { verify_integrity(resized_bytes_1[:min(start, target)], seed) if target > start { verify_zeroed(resized_bytes_1[start:]) + randomize_bytes(resized_bytes_1[start:], seed) } resized_bytes_2, resize_2_err := allocator.procedure(allocator.data, .Resize, start, alignment, raw_data(resized_bytes_1), target) @@ -485,10 +410,10 @@ test_parallel_pointer_passing :: proc(thread_count: int) { } /* -This test makes sure that a Superpage is reused when abandoned by a thread and +This test makes sure that a Segment is reused when abandoned by a thread and picked up by a different one. */ -test_superpage_abandonment_and_reuse :: proc() { +test_segment_abandonment_and_reuse :: proc() { Alloc_Data :: struct { thread: ^thread.Thread, slice: []int, @@ -498,10 +423,10 @@ test_superpage_abandonment_and_reuse :: proc() { alloc_task :: proc(t: ^thread.Thread) { // In this first thread, we allocate a small chunk of memory in a new - // superpage and mark where it came from. + // segment and mark where it came from. data := cast(^Alloc_Data)t.data - data.slice = make([]int, 4096) - data.signature = runtime.find_superpage_from_pointer(raw_data(data.slice)) + data.slice = make([]int, 256) + data.signature = runtime.find_segment_from_pointer(raw_data(data.slice)) for &v, i in data.slice { v = i } @@ -518,15 +443,16 @@ test_superpage_abandonment_and_reuse :: proc() { } // Delete the data and allocate a new integer. If everything works as // expected, this thread will have remotely freed the old chunk of - // memory, then adopted the superpage with the new operation. + // memory, then adopted the segment with the new operation. // // Upon adoption, the remote free should be acknowledged. delete(data.slice) - x := new(int) - free(x) + x := make([]int, 256) + defer delete(x) // This is where we check to make sure the new pointer comes from the // same place as the old data. - expect(runtime.find_superpage_from_pointer(x) == data.signature) + signature := runtime.find_segment_from_pointer(raw_data(x)) + expect(signature == data.signature) sync.post(&data.done) } @@ -539,7 +465,7 @@ test_superpage_abandonment_and_reuse :: proc() { sync.wait(&data.done) - // It will take an infinitesimal amount of time for the superpage to be + // It will take an infinitesimal amount of time for the segment to be // pushed to the orphanage, so let's wait a (rather long) moment. time.sleep(1 * time.Millisecond) @@ -555,7 +481,7 @@ test_superpage_abandonment_and_reuse :: proc() { thread.join(reuser) thread.destroy(allocer) thread.destroy(reuser) - log.info("Superpage abandonment and reuse test succeeded.") + log.info("Segment abandonment and reuse test succeeded.") } /* @@ -590,9 +516,9 @@ test_parallel_pointer_resizing :: proc(thread_count: int) { resized_ptr, resize_err := allocator.procedure(allocator.data, .Resize, new_len, 1, data.ptr^, old_len) expect(resize_err == nil) - // If we're dealing with sub-slab sizes, the pointer should stay the + // If we're dealing with Small/Large sizes, the pointer should stay the // same if the bin rank did not change. - if new_len <= runtime.HEAP_SLAB_SIZE { + if new_len <= runtime.ODIN_HEAP_MAX_BIN_SIZE { old_rank := runtime.heap_bin_size_to_rank(runtime.heap_round_to_bin_size(old_len)) new_rank := runtime.heap_bin_size_to_rank(runtime.heap_round_to_bin_size(new_len)) @@ -650,7 +576,7 @@ test_parallel_pointer_resizing :: proc(thread_count: int) { log.info("Parallel pointer resize test succeeded.") } -test_orphaned_superpage_with_remote_frees :: proc() { +test_orphaned_segment_with_remote_frees :: proc() { Data :: struct { thread: ^thread.Thread, ptr: ^int, @@ -678,66 +604,14 @@ test_orphaned_superpage_with_remote_frees :: proc() { thread.join(data.thread) thread.destroy(data.thread) - log.info("Oprhaned superpage with remote free test succeeded.") -} - -test_orphanage_overflow :: proc(thread_count: int) { - Data :: struct { - thread: ^thread.Thread, - barrier: ^sync.Barrier, - wg: ^sync.Wait_Group, - } - - task :: proc(t: ^thread.Thread) { - data := cast(^Data)t.data - - x := new(int) - intrinsics.mem_zero_volatile(x, size_of(int)) - sync.barrier_wait(data.barrier) - - free(x) - - sync.wait_group_done(data.wg) - } - - tasks := make([]Data, thread_count, context.temp_allocator) - - barrier: sync.Barrier - sync.barrier_init(&barrier, thread_count) - - wg: sync.Wait_Group - sync.wait_group_add(&wg, thread_count) - - for i in 0.. 0) - expect(slab.bin_size > 0) - expect(intrinsics.atomic_load_explicit(&slab.remote_free_bins_scheduled, .Acquire) == 0) - - list := make([]^int, slab.max_bins) - list[0] = o - - for i in 0.. Large + test_single_alloc_and_resize(runtime.ODIN_HEAP_MIN_BIN_SIZE, 1 + runtime.ODIN_HEAP_SMALL_BIN_MAX) + test_single_alloc_and_resize(1 + runtime.ODIN_HEAP_SMALL_BIN_MAX, runtime.ODIN_HEAP_MIN_BIN_SIZE) - // Cross-category tests. - // Bin <-> Slab - test_single_alloc_and_resize(runtime.HEAP_MAX_BIN_SIZE, runtime.HEAP_SLAB_SIZE) - test_single_alloc_and_resize(runtime.HEAP_SLAB_SIZE, runtime.HEAP_MAX_BIN_SIZE) + // Small <-> Huge + test_single_alloc_and_resize(runtime.ODIN_HEAP_MIN_BIN_SIZE, 1 + runtime.ODIN_HEAP_MAX_BIN_SIZE) + test_single_alloc_and_resize(1 + runtime.ODIN_HEAP_MAX_BIN_SIZE, runtime.ODIN_HEAP_MIN_BIN_SIZE) - // Bin <-> Huge - test_single_alloc_and_resize(runtime.HEAP_MAX_BIN_SIZE, runtime.HEAP_HUGE_ALLOCATION_THRESHOLD) - test_single_alloc_and_resize(runtime.HEAP_HUGE_ALLOCATION_THRESHOLD, runtime.HEAP_MAX_BIN_SIZE) - - // Slab <-> Huge - test_single_alloc_and_resize(runtime.HEAP_MAX_BIN_SIZE + 1, runtime.HEAP_HUGE_ALLOCATION_THRESHOLD) - test_single_alloc_and_resize(runtime.HEAP_HUGE_ALLOCATION_THRESHOLD, runtime.HEAP_MAX_BIN_SIZE + 1) - - // Inter-huge tests. - test_single_alloc_and_resize(runtime.HEAP_HUGE_ALLOCATION_THRESHOLD + 2, runtime.HEAP_HUGE_ALLOCATION_THRESHOLD + 1) - test_single_alloc_and_resize(runtime.HEAP_HUGE_ALLOCATION_THRESHOLD + 1, runtime.SUPERPAGE_SIZE) - - // Larger-than-superpage tests. - test_single_alloc_and_resize(runtime.SUPERPAGE_SIZE, runtime.SUPERPAGE_SIZE * 3) - test_single_alloc_and_resize(runtime.SUPERPAGE_SIZE * 3, runtime.SUPERPAGE_SIZE) + // Large <-> Huge + test_single_alloc_and_resize(1 + runtime.ODIN_HEAP_MAX_BIN_SIZE, 1 + runtime.ODIN_HEAP_SMALL_BIN_MAX) + test_single_alloc_and_resize(1 + runtime.ODIN_HEAP_SMALL_BIN_MAX, 1 + runtime.ODIN_HEAP_MAX_BIN_SIZE) // Brute-force tests. - test_individual_allocation_and_free(runtime.HEAP_MAX_BIN_SIZE if opt.long else 1024) - test_continuous_allocation_of_size_n(16, runtime.HEAP_MAX_BIN_SIZE if opt.long else 1024) + test_individual_allocation_and_free(runtime.ODIN_HEAP_MAX_BIN_SIZE if opt.long else 1024) + test_continuous_allocation_of_size_n(16, runtime.ODIN_HEAP_MAX_BIN_SIZE if opt.long else 1024) test_alloc_write_free( object_count = 400, @@ -1342,36 +1075,12 @@ main :: proc() { free_direction = .Forward, ) - test_alloc_write_free( - object_count = runtime.HEAP_SLAB_COUNT*2, - starting_size = runtime.HEAP_SLAB_SIZE/2, final_size = runtime.HEAP_SLAB_SIZE*4, - size_strategy = .Multiplying, size_operand = 2, - allocs_per_free_operation = 1, - free_operations_at_once = 1, - free_strategy = .At_The_End, - free_direction = .Forward, - ) - - test_alloc_write_free( - object_count = 2, - starting_size = runtime.SUPERPAGE_SIZE/2, final_size = runtime.SUPERPAGE_SIZE*4, - size_strategy = .Multiplying, size_operand = 2, - allocs_per_free_operation = 1, - free_operations_at_once = 1, - free_strategy = .At_The_End, - free_direction = .Forward, - ) - // This is a lengthy test and won't tell us much more than any other test will. if opt.long { - test_single_alloc_and_resize_incremental(0, runtime.HEAP_SLAB_SIZE) + test_single_alloc_and_resize_incremental(0, runtime.ODIN_HEAP_MAX_BIN_SIZE) } - runtime.compact_heap() - - test_serial_bin_sanity() - - runtime.compact_heap() + runtime.compact_local_heap() } if opt.serial_benchmarks { @@ -1384,8 +1093,9 @@ main :: proc() { bench_alloc_n_then_free_n(10_000_000, Struct_64) bench_alloc_n_then_free_n(10_000_000, Struct_512) bench_alloc_n_then_free_n(100_000, [8192]u8) - bench_alloc_n_then_free_n(10_000, [runtime.HEAP_SLAB_SIZE/4]u8) - bench_alloc_n_then_free_n(10_000, [runtime.HEAP_SLAB_SIZE*4]u8) + bench_alloc_n_then_free_n(100_000, [4096*4]u8) + bench_alloc_n_then_free_n(10_000, [65536/4]u8) + bench_alloc_n_then_free_n(10_000, [65536*4]u8) bench_alloc_n_then_free_n(100, [runtime.SUPERPAGE_SIZE]u8) log.info("* Freeing backwards ...") @@ -1395,8 +1105,8 @@ main :: proc() { bench_alloc_n_then_free_n_backwards(10_000_000, Struct_64) bench_alloc_n_then_free_n_backwards(10_000_000, Struct_512) bench_alloc_n_then_free_n_backwards(100_000, [8192]u8) - bench_alloc_n_then_free_n_backwards(10_000, [runtime.HEAP_SLAB_SIZE/4]u8) - bench_alloc_n_then_free_n_backwards(10_000, [runtime.HEAP_SLAB_SIZE*4]u8) + bench_alloc_n_then_free_n_backwards(10_000, [65536/4]u8) + bench_alloc_n_then_free_n_backwards(10_000, [65536*4]u8) bench_alloc_n_then_free_n_backwards(100, [runtime.SUPERPAGE_SIZE]u8) log.info("* Freeing randomly ...") @@ -1406,10 +1116,9 @@ main :: proc() { bench_alloc_n_then_free_n_randomly(10_000_000, Struct_64) bench_alloc_n_then_free_n_randomly(10_000_000, Struct_512) bench_alloc_n_then_free_n_randomly(100_000, [8192]u8) - bench_alloc_n_then_free_n_randomly(100_000, [runtime.HEAP_SLAB_SIZE/4]u8) - bench_alloc_n_then_free_n_randomly(100_000, [runtime.HEAP_SLAB_SIZE-runtime.HEAP_SLAB_ALLOCATION_BOOK_KEEPING]u8) - bench_alloc_n_then_free_n_randomly(100_000, [runtime.HEAP_SLAB_SIZE]u8) - bench_alloc_n_then_free_n_randomly(100_000, [runtime.HEAP_SLAB_SIZE*2]u8) + bench_alloc_n_then_free_n_randomly(100_000, [65536/4]u8) + bench_alloc_n_then_free_n_randomly(100_000, [65536]u8) + bench_alloc_n_then_free_n_randomly(100_000, [65536*2]u8) bench_alloc_n_then_free_n_randomly(100, [runtime.SUPERPAGE_SIZE]u8) log.info("* Allocating and freeing repeatedly ...") @@ -1444,7 +1153,7 @@ main :: proc() { bench_1_producer_n_consumer_for_m_alloc(2, 10_000, [8192]u8) bench_1_producer_n_consumer_for_m_alloc(4, 10_000, [8192]u8) - bench_1_producer_n_consumer_for_m_alloc(4, 100, [runtime.HEAP_SLAB_SIZE]u8) + bench_1_producer_n_consumer_for_m_alloc(4, 100, [65536]u8) when .Thread not_in ODIN_SANITIZER_FLAGS { // NOTE: TSan doesn't work well with excessive thread counts, @@ -1460,7 +1169,7 @@ main :: proc() { log.info("Tests complete.") if opt.compact { - runtime.compact_heap() + runtime.compact_local_heap() log.info("The main thread's heap has been compacted.") } @@ -1478,15 +1187,18 @@ main :: proc() { } } + // for ptr, entry in tracker.allocation_map { + // log.infof("%p -- %v", ptr, entry) + // } + // mem.tracking_allocator_destroy(&tracker) + + runtime.heap_release_empty_orphans() + if opt.info { heap_info := runtime.get_local_heap_info() log.infof("%#v", heap_info) } - // if .Dump_Slabs in params { - // dump_slabs() - // } - if opt.trap { intrinsics.debug_trap() }