diff --git a/base/runtime/heap_allocator_implementation.odin b/base/runtime/heap_allocator_implementation.odin index 8c0b39708..e2186691e 100644 --- a/base/runtime/heap_allocator_implementation.odin +++ b/base/runtime/heap_allocator_implementation.odin @@ -133,6 +133,7 @@ ODIN_HEAP_SMALL_BIN_MAX :: #config(ODIN_HEAP_SMALL_BIN_MAX, 8 * Kilobyte) // [ // Constants // +ODIN_HEAP_SEGMENT_SIZE :: 4 * Megabyte ODIN_HEAP_MIN_BIN_SHIFT :: intrinsics.constant_log2(ODIN_HEAP_MIN_BIN_SIZE) ODIN_HEAP_MAX_BIN_SHIFT :: intrinsics.constant_log2(ODIN_HEAP_MAX_BIN_SIZE) ODIN_HEAP_BIN_RANKS :: 1 + ODIN_HEAP_MAX_BIN_SHIFT - ODIN_HEAP_MIN_BIN_SHIFT @@ -197,7 +198,7 @@ Heap_Debug_Level :: enum { Heap_Slab_Class :: enum { Small, // Slabs are `ODIN_HEAP_SMALL_SLAB_SIZE` (64KiB) each. - Large, // One segment-wide (~2MiB) slab. + Large, // One segment-wide (platform-dependent size) slab. Huge, // One slab for one allocation, sized specifically for the request. } @@ -221,7 +222,12 @@ Allocate a new Segment that may be used to store either Small or Large slabs. @(require_results) heap_allocate_segment :: #force_inline proc "contextless" () -> ^Heap_Segment { when ODIN_HEAP_SEGMENT_SIZE_OVERRIDE == 0 { - return cast(^Heap_Segment)allocate_virtual_memory_superpage() + if superpage_size != 0 { + return cast(^Heap_Segment)allocate_virtual_memory_superpage() + } else { + // Use the default segment value. + return cast(^Heap_Segment)allocate_virtual_memory_aligned(ODIN_HEAP_SEGMENT_SIZE, ODIN_HEAP_SEGMENT_SIZE) + } } else { return cast(^Heap_Segment)allocate_virtual_memory_aligned(ODIN_HEAP_SEGMENT_SIZE_OVERRIDE, ODIN_HEAP_SEGMENT_SIZE_OVERRIDE) } @@ -233,8 +239,11 @@ Get the constant size for all segments. This size also dictates each segment's a @(require_results) heap_get_segment_size :: #force_inline proc "contextless" () -> int { when ODIN_HEAP_SEGMENT_SIZE_OVERRIDE == 0 { - // TODO: Derive from the OS config. - return SUPERPAGE_SIZE + if size := superpage_size; size != 0 { + return size + } else { + return ODIN_HEAP_SEGMENT_SIZE + } } else { return ODIN_HEAP_SEGMENT_SIZE_OVERRIDE } @@ -426,8 +435,6 @@ The **Segment** is a single contiguous allocation from the operating system's virtual memory subsystem, subdivided into Slabs. All metadata lives at the head of the allocation. -On almost every platform, this structure will be 2MiB by default. - Depending on the operating system, addresses within the space occupied by the Segment (and hence its allocations) may also have faster access times due to leveraging properties of the Translation Lookaside Buffer. diff --git a/base/runtime/os_specific_sysv.odin b/base/runtime/os_specific_sysv.odin new file mode 100644 index 000000000..d50cec085 --- /dev/null +++ b/base/runtime/os_specific_sysv.odin @@ -0,0 +1,61 @@ +#+private +#+build linux, freebsd, netbsd, openbsd +package runtime + +// See the System Five Application Binary Interface ยง 3.4.3 for more information. + +// Figure 3.11: Auxiliary Vector Types +Auxiliary_Vector_Type :: enum i32 { + AT_NULL = 0, // ignored + AT_IGNORE = 1, // ignored + AT_EXECFD = 2, // a_val + AT_PHDR = 3, // a_ptr + AT_PHENT = 4, // a_val + AT_PHNUM = 5, // a_val + AT_PAGESZ = 6, // a_val + AT_BASE = 7, // a_ptr + AT_FLAGS = 8, // a_val + AT_ENTRY = 9, // a_ptr + AT_NOTELF = 10, // a_val + AT_UID = 11, // a_val + AT_EUID = 12, // a_val + AT_GID = 13, // a_val + AT_EGID = 14, // a_val +} + +@(private="file") +c_long :: i32 when size_of(rawptr) == 4 else i64 + +// Figure 3.10: auxv_t Type Definition +auxv_t :: struct { + a_type: Auxiliary_Vector_Type, + using a_un: struct #raw_union { + a_val: c_long, // long + a_ptr: rawptr, // void* + a_fnc: proc "c" (), // void (*)() + }, +} + +auxv__: [^]auxv_t + +// Mind the alphanumeric sorted naming of the files in `base:runtime`, as this +// init needs to run before the virtual memory init. +@(init) +init_auxv :: proc "contextless" () { + // This is similar to how we get the environment on Linux. + #no_bounds_check auxv := cast([^]rawptr)&args__[len(args__) + 1] + for auxv[0] != nil { + auxv = auxv[1:] + } + auxv__ = cast([^]auxv_t)(auxv[1:]) +} + +// Get a value from the auxiliary vector. +_get_auxiliary :: proc "contextless" (at: Auxiliary_Vector_Type) -> (value: auxv_t, found: bool) { + for ap := auxv__; ap != nil && ap[0].a_type != .AT_NULL; ap = ap[1:] { + if ap[0].a_type == at { + return ap[0], true + } + } + return +} diff --git a/base/runtime/virtual_memory.odin b/base/runtime/virtual_memory.odin index e4116d260..b2c14a9e3 100644 --- a/base/runtime/virtual_memory.odin +++ b/base/runtime/virtual_memory.odin @@ -1,21 +1,26 @@ package runtime +import "base:intrinsics" + ODIN_VIRTUAL_MEMORY_SUPPORTED :: VIRTUAL_MEMORY_SUPPORTED -// Virtually all MMUs supported by Odin should have a 4KiB page size. -PAGE_SIZE :: 4 * Kilobyte +/* +The page size of the operating system, used for virtual memory allocations. +*/ +page_size: int -when ODIN_ARCH == .arm32 { - SUPERPAGE_SIZE :: 1 * Megabyte -} else { - // All other architectures should have support for 2MiB pages. - // i386 supports it in PAE mode. - // amd64, arm64, and riscv64 support it by default. - SUPERPAGE_SIZE :: 2 * Megabyte +/* +The superpage size of the operating system. + +This may be zero if unavailable. +*/ +superpage_size: int + +@(init, private) +init_virtual_memory :: proc "contextless" () { + _init_virtual_memory() } -#assert(SUPERPAGE_SIZE & (SUPERPAGE_SIZE-1) == 0, "SUPERPAGE_SIZE must be a power of two.") - /* Allocate virtual memory from the operating system. @@ -35,14 +40,20 @@ This is a contiguous block of memory larger than what is normally distributed by the operating system, sometimes with special performance properties related to the Translation Lookaside Buffer. -The address will be a multiple of the `SUPERPAGE_SIZE` constant, and the memory -pointed to will be at least as long as that very same constant. +The address will be a multiple of `superpage_size`, and the memory +pointed to will be at least as long as that. The name derives from the superpage concept on the *BSD operating systems, where it is known as huge pages on Linux and large pages on Windows. + +This may return nil if a superpage size was unable to be retrieved from the +operating system or if the feature is otherwise unavailable. */ @(require_results) allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr { + if superpage_size == 0 { + return nil + } return _allocate_virtual_memory_superpage() } diff --git a/base/runtime/virtual_memory_darwin.odin b/base/runtime/virtual_memory_darwin.odin index 77b46dc82..39160fcb5 100644 --- a/base/runtime/virtual_memory_darwin.odin +++ b/base/runtime/virtual_memory_darwin.odin @@ -8,6 +8,7 @@ VIRTUAL_MEMORY_SUPPORTED :: true foreign import lib "system:System.framework" foreign lib { + vm_page_size: uintptr mach_task_self_: u32 mach_vm_allocate :: proc(target: u32, address: ^u64, size: u64, flags: i32) -> i32 --- mach_vm_deallocate :: proc(target: u32, address: u64, size: u64) -> i32 --- @@ -55,6 +56,25 @@ VM_PROT_READ :: 0x01 VM_PROT_WRITE :: 0x02 VM_INHERIT_COPY :: 1 +_init_virtual_memory :: proc "contextless" () { + page_size = _get_page_size() + superpage_size = _get_superpage_size() +} + +_get_page_size :: proc "contextless" () -> int { + return int(vm_page_size) +} + +_get_superpage_size :: proc "contextless" () -> int { + when ODIN_ARCH == .amd64 { + // NOTE(Feoramund): As far as we are aware, Darwin only supports + // explicit superpage allocation on AMD64 with a 2MiB parameter. + return 2 * Megabyte + } else { + return 0 + } +} + _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { address: u64 result := mach_vm_map(mach_task_self_, &address, u64(size), 0, VM_FLAGS_ANYWHERE, MEMORY_OBJECT_NULL, 0, false, VM_PROT_READ|VM_PROT_WRITE, VM_PROT_READ|VM_PROT_WRITE, VM_INHERIT_COPY) @@ -66,22 +86,14 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { _allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr { address: u64 - flags: i32 = VM_FLAGS_ANYWHERE - when ODIN_ARCH == .amd64 { - // NOTE(Feoramund): As far as we are aware, Darwin only supports - // explicit superpage allocation on AMD64 with a 2MiB parameter. - when SUPERPAGE_SIZE == 2 * Megabyte { - flags |= VM_FLAGS_SUPERPAGE_SIZE_2MB - } else { - #panic("An unsupported superpage size has been configured for AMD64 Darwin; only 2MB is supported.") - } - } - alignment_mask: u64 = SUPERPAGE_SIZE - 1 // Assumes a power of two size, ensured by an assertion in `virtual_memory.odin`. - result := mach_vm_map(mach_task_self_, &address, SUPERPAGE_SIZE, alignment_mask, flags, MEMORY_OBJECT_NULL, 0, false, VM_PROT_READ|VM_PROT_WRITE, VM_PROT_READ|VM_PROT_WRITE, VM_INHERIT_COPY) + flags: i32 = VM_FLAGS_ANYWHERE | VM_FLAGS_SUPERPAGE_SIZE_2MB + assert_contextless(superpage_size & (superpage_size-1) == 0, "The superpage size is not a power of two.") + alignment_mask: u64 = u64(superpage_size) - 1 + result := mach_vm_map(mach_task_self_, &address, 2 * Megabyte, alignment_mask, flags, MEMORY_OBJECT_NULL, 0, false, VM_PROT_READ|VM_PROT_WRITE, VM_PROT_READ|VM_PROT_WRITE, VM_INHERIT_COPY) if result != 0 { return nil } - assert_contextless(address % SUPERPAGE_SIZE == 0) + assert_contextless(address % u64(superpage_size) == 0) return rawptr(uintptr(address)) } diff --git a/base/runtime/virtual_memory_freebsd.odin b/base/runtime/virtual_memory_freebsd.odin index 76acfeebd..4d7c04600 100644 --- a/base/runtime/virtual_memory_freebsd.odin +++ b/base/runtime/virtual_memory_freebsd.odin @@ -25,7 +25,46 @@ MAP_ANONYMOUS :: 0x1000 MAP_ALIGNMENT_SHIFT :: 24 MAP_ALIGNED_SUPER :: 1 << MAP_ALIGNMENT_SHIFT -SUPERPAGE_MAP_FLAGS :: (intrinsics.constant_log2(SUPERPAGE_SIZE) << MAP_ALIGNMENT_SHIFT) | MAP_ALIGNED_SUPER +_init_virtual_memory :: proc "contextless" () { + page_size = _get_page_size() + superpage_size = _get_superpage_size() +} + +_get_page_size :: proc "contextless" () -> int { + // This is a fallback value if the auxiliary vector does not supply it. + DEFAULT_PAGE_SIZE :: 4096 + + if value, found := _get_auxiliary(.AT_PAGESZ); found { + return int(value.a_val) + } else { + return DEFAULT_PAGE_SIZE + } +} + +_get_superpage_size :: proc "contextless" () -> int { + // This is specific to FreeBSD and not defined in the SysV ABI. + AT_PAGESIZES :: Auxiliary_Vector_Type(20) + + if value, found := _get_auxiliary(AT_PAGESIZES); found { + greatest_size := 0 + supports_2mib := false + + for sizes := cast([^]uint)value.a_ptr; sizes[0] != 0; sizes = sizes[1:] { + if sizes[0] == 2 * Megabyte { + // The standard 2MiB superpage is supported. + supports_2mib = true + } + greatest_size = max(greatest_size, int(sizes[0])) + } + + if supports_2mib { + return 2 * Megabyte + } else if greatest_size > _get_page_size() { + return greatest_size + } + } + return 0 +} _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0) @@ -36,11 +75,13 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { } _allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr { - result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, SUPERPAGE_SIZE, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|SUPERPAGE_MAP_FLAGS, ~uintptr(0), 0) + superpage_flags := uintptr(intrinsics.count_trailing_zeros(superpage_size) << MAP_ALIGNMENT_SHIFT) | MAP_ALIGNED_SUPER + + result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(superpage_size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|superpage_flags, ~uintptr(0), 0) if !ok { // It may be the case that FreeBSD couldn't fulfill our alignment // request, but it could still give us some memory. - return _allocate_virtual_memory_manually_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE) + return _allocate_virtual_memory_manually_aligned(superpage_size, superpage_size) } return rawptr(result) } @@ -49,7 +90,7 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in // This procedure uses the `MAP_ALIGNED` API provided by FreeBSD and falls // back to manually aligned addresses, if that fails. map_aligned_n: uintptr - if alignment >= PAGE_SIZE { + if alignment >= page_size { map_aligned_n = intrinsics.count_trailing_zeros(uintptr(alignment)) << MAP_ALIGNMENT_SHIFT } result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|map_aligned_n, ~uintptr(0), 0) @@ -60,7 +101,7 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in } _allocate_virtual_memory_manually_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr { - if alignment <= PAGE_SIZE { + if alignment <= page_size { // This is the simplest case. // // By virtue of binary arithmetic, any address aligned to a power of @@ -78,7 +119,7 @@ _allocate_virtual_memory_manually_aligned :: proc "contextless" (size: int, alig if !ok { return nil } - assert_contextless(mmap_result % PAGE_SIZE == 0) + assert_contextless(mmap_result % uintptr(page_size) == 0) modulo := mmap_result & uintptr(alignment-1) if modulo != 0 { // The address is misaligned, so we must return an adjusted address @@ -89,24 +130,24 @@ _allocate_virtual_memory_manually_aligned :: proc "contextless" (size: int, alig // Sanity-checking: // - The adjusted address is still page-aligned, so it is a valid argument for munmap. // - The adjusted address is aligned to the user's needs. - assert_contextless(adjusted_result % PAGE_SIZE == 0) + assert_contextless(adjusted_result % uintptr(page_size) == 0) assert_contextless(adjusted_result % uintptr(alignment) == 0) // Round the delta to a multiple of the page size. - delta = delta / PAGE_SIZE * PAGE_SIZE + delta = delta / uintptr(page_size) * uintptr(page_size) if delta > 0 { // Unmap the pages we don't need. intrinsics.syscall_bsd(SYS_munmap, mmap_result, delta) } return rawptr(adjusted_result) - } else if size + alignment > PAGE_SIZE { + } else if size + alignment > page_size { // The address is coincidentally aligned as desired, but we have space // that will never be seen by the user, so we must free the backing // pages for it. - start := size / PAGE_SIZE * PAGE_SIZE - if size % PAGE_SIZE != 0 { - start += PAGE_SIZE + start := size / page_size * page_size + if size % page_size != 0 { + start += page_size } length := size + alignment - start if length > 0 { diff --git a/base/runtime/virtual_memory_linux.odin b/base/runtime/virtual_memory_linux.odin index 94eea7a13..8d3f48e54 100644 --- a/base/runtime/virtual_memory_linux.odin +++ b/base/runtime/virtual_memory_linux.odin @@ -6,25 +6,45 @@ import "base:intrinsics" VIRTUAL_MEMORY_SUPPORTED :: true when ODIN_ARCH == .amd64 { - SYS_mmap :: uintptr(9) - SYS_munmap :: uintptr(11) - SYS_mremap :: uintptr(25) + SYS_open :: uintptr(2) + SYS_read :: uintptr(0) + SYS_close :: uintptr(3) + + SYS_mmap :: uintptr(9) + SYS_munmap :: uintptr(11) + SYS_mremap :: uintptr(25) } else when ODIN_ARCH == .arm32 { - SYS_mmap :: uintptr(90) - SYS_munmap :: uintptr(91) - SYS_mremap :: uintptr(163) + SYS_open :: uintptr(5) + SYS_read :: uintptr(3) + SYS_close :: uintptr(6) + + SYS_mmap :: uintptr(90) + SYS_munmap :: uintptr(91) + SYS_mremap :: uintptr(163) } else when ODIN_ARCH == .arm64 { - SYS_mmap :: uintptr(222) - SYS_munmap :: uintptr(215) - SYS_mremap :: uintptr(216) + SYS_openat :: uintptr(56) + SYS_read :: uintptr(63) + SYS_close :: uintptr(57) + + SYS_mmap :: uintptr(222) + SYS_munmap :: uintptr(215) + SYS_mremap :: uintptr(216) } else when ODIN_ARCH == .i386 { - SYS_mmap :: uintptr(90) - SYS_munmap :: uintptr(91) - SYS_mremap :: uintptr(163) + SYS_open :: uintptr(5) + SYS_read :: uintptr(3) + SYS_close :: uintptr(6) + + SYS_mmap :: uintptr(90) + SYS_munmap :: uintptr(91) + SYS_mremap :: uintptr(163) } else when ODIN_ARCH == .riscv64 { - SYS_mmap :: uintptr(222) - SYS_munmap :: uintptr(215) - SYS_mremap :: uintptr(216) + SYS_openat :: uintptr(56) + SYS_read :: uintptr(63) + SYS_close :: uintptr(57) + + SYS_mmap :: uintptr(222) + SYS_munmap :: uintptr(215) + SYS_mremap :: uintptr(216) } else { #panic("Syscall numbers related to virtual memory are missing for this Linux architecture.") } @@ -39,6 +59,105 @@ MREMAP_MAYMOVE :: 0x01 ENOMEM :: ~uintptr(11) +_init_virtual_memory :: proc "contextless" () { + page_size = _get_page_size() + superpage_size = _get_superpage_size() +} + +_get_page_size :: proc "contextless" () -> int { + // This is a fallback value if the auxiliary vector does not supply it. + DEFAULT_PAGE_SIZE :: 4096 + + if value, found := _get_auxiliary(.AT_PAGESZ); found { + return int(value.a_val) + } else { + return DEFAULT_PAGE_SIZE + } +} + +_get_superpage_size :: proc "contextless" () -> int { + meminfo: cstring = "/proc/meminfo" + + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { + AT_FDCWD :: ~uintptr(99) // -100 + fd := cast(int)intrinsics.syscall(SYS_openat, AT_FDCWD, transmute(uintptr)meminfo, 0 /* flags */, 0 /* mode */) + } else { + fd := cast(int)intrinsics.syscall(SYS_open, transmute(uintptr)meminfo, 0 /* flags */, 0 /* mode */) + } + if fd < 0 { + // Error on opening file. + return 0 + } + defer intrinsics.syscall(SYS_close, uintptr(fd)) + + buf: [4096]u8 + read := cast(int)intrinsics.syscall(SYS_read, cast(uintptr)fd, cast(uintptr)&buf[0], len(buf)) + if read <= 0 { + // Failed to read anything. + return 0 + } + + // Parse the file. It's in a format of "KEY: VALUE\n" with a + // variable number of spaces after the colon. + str := buf[:read] + for len(str) > 0 { + key, val: []u8 + // Get the key. + for c, i in str { + if c == ':' { + key, str = str[:i], str[1+i:] + break + } + } + // Trim the spaces. + for c, i in str { + if c != ' ' { + str = str[i:] + break + } + } + // Get the value. + for c, i in str { + if c == '\n' { + val, str = str[:i], str[1+i:] + break + } + } + // Break in the event something was parsed incorrectly. + if len(key) == 0 || len(val) == 0 { + break + } + + if string(key) == "Hugepagesize" { + // The value will be in a format like: 2048 kB + n, unit: []u8 + for c, i in val { + if c == ' ' { + n = val[:i] + unit = val[1+i:] + break + } + } + // Convert it to a number. + bytes := 0 + for c in n { + bytes *= 10 + bytes += int(c - '0') + } + // The man page for `proc_meminfo` does not state if it + // uses measurements other than "kB" but just to be safe. + switch string(unit) { + case "kB": bytes *= Kilobyte + case "mB": bytes *= Megabyte + case "gB": bytes *= Gigabyte + } + + return bytes + } + } + return 0 +} + _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { result := intrinsics.syscall(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0) if int(result) < 0 { @@ -49,22 +168,22 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { _allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr { // This depends on Transparent HugePage Support being enabled. - result := intrinsics.syscall(SYS_mmap, 0, SUPERPAGE_SIZE, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0) + result := intrinsics.syscall(SYS_mmap, 0, uintptr(superpage_size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0) if int(result) < 0 { return nil } - if uintptr(result) % SUPERPAGE_SIZE != 0 { + if uintptr(result) % uintptr(superpage_size) != 0 { // If THP support is not enabled, we may receive an address aligned to a // page boundary instead, in which case, we must manually align a new // address. - _free_virtual_memory(rawptr(result), SUPERPAGE_SIZE) - return _allocate_virtual_memory_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE) + _free_virtual_memory(rawptr(result), superpage_size) + return _allocate_virtual_memory_aligned(superpage_size, superpage_size) } return rawptr(result) } _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr { - if alignment <= PAGE_SIZE { + if alignment <= page_size { // This is the simplest case. // // By virtue of binary arithmetic, any address aligned to a power of @@ -82,7 +201,7 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in if int(mmap_result) < 0 { return nil } - assert_contextless(mmap_result % PAGE_SIZE == 0) + assert_contextless(mmap_result % uintptr(page_size) == 0) modulo := mmap_result & uintptr(alignment-1) if modulo != 0 { // The address is misaligned, so we must return an adjusted address @@ -93,24 +212,24 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in // Sanity-checking: // - The adjusted address is still page-aligned, so it is a valid argument for mremap and munmap. // - The adjusted address is aligned to the user's needs. - assert_contextless(adjusted_result % PAGE_SIZE == 0) + assert_contextless(adjusted_result % uintptr(page_size) == 0) assert_contextless(adjusted_result % uintptr(alignment) == 0) // Round the delta to a multiple of the page size. - delta = delta / PAGE_SIZE * PAGE_SIZE + delta = delta / uintptr(page_size) * uintptr(page_size) if delta > 0 { // Unmap the pages we don't need. intrinsics.syscall(SYS_munmap, mmap_result, delta) } return rawptr(adjusted_result) - } else if size + alignment > PAGE_SIZE { + } else if size + alignment > page_size { // The address is coincidentally aligned as desired, but we have space // that will never be seen by the user, so we must free the backing // pages for it. - start := size / PAGE_SIZE * PAGE_SIZE - if size % PAGE_SIZE != 0 { - start += PAGE_SIZE + start := size / page_size * page_size + if size % page_size != 0 { + start += page_size } length := size + alignment - start if length > 0 { diff --git a/base/runtime/virtual_memory_netbsd.odin b/base/runtime/virtual_memory_netbsd.odin index 46c475d9a..1e6ccbef9 100644 --- a/base/runtime/virtual_memory_netbsd.odin +++ b/base/runtime/virtual_memory_netbsd.odin @@ -23,6 +23,26 @@ MAP_ANONYMOUS :: 0x1000 // #define MAP_ALIGNED(n) ((int)((unsigned int)(n) << MAP_ALIGNMENT_SHIFT)) MAP_ALIGNMENT_SHIFT :: 24 +_init_virtual_memory :: proc "contextless" () { + page_size = _get_page_size() +} + +_get_page_size :: proc "contextless" () -> int { + // This is a fallback value if the auxiliary vector does not supply it. + DEFAULT_PAGE_SIZE :: 4096 + + if value, found := _get_auxiliary(.AT_PAGESZ); found { + return int(value.a_val) + } else { + return DEFAULT_PAGE_SIZE + } +} + +_get_superpage_size :: proc "contextless" () -> int { + // NOTE(Feoramund): I am uncertain if NetBSD has direct support for superpages. + return 0 +} + _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0) if !ok { @@ -32,9 +52,7 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { } _allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr { - // NOTE(Feoramund): I am uncertain if NetBSD has direct support for - // superpages, so we just use the aligned allocate procedure here. - return _allocate_virtual_memory_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE) + return nil } _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr { diff --git a/base/runtime/virtual_memory_openbsd.odin b/base/runtime/virtual_memory_openbsd.odin index 6a37d4ae6..f9657c21f 100644 --- a/base/runtime/virtual_memory_openbsd.odin +++ b/base/runtime/virtual_memory_openbsd.odin @@ -14,6 +14,26 @@ PROT_WRITE :: 0x02 MAP_PRIVATE :: 0x0002 MAP_ANONYMOUS :: 0x1000 +_init_virtual_memory :: proc "contextless" () { + page_size = _get_page_size() +} + +_get_page_size :: proc "contextless" () -> int { + // This is a fallback value if the auxiliary vector does not supply it. + DEFAULT_PAGE_SIZE :: 4096 + + if value, found := _get_auxiliary(.AT_PAGESZ); found { + return int(value.a_val) + } else { + return DEFAULT_PAGE_SIZE + } +} + +_get_superpage_size :: proc "contextless" () -> int { + // NOTE(Feoramund): I am uncertain if OpenBSD has direct support for superpages. + return 0 +} + _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0) if !ok { @@ -23,13 +43,11 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { } _allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr { - // NOTE(Feoramund): I am uncertain if OpenBSD has direct support for - // superpages, so we just use the aligned allocate procedure here. - return _allocate_virtual_memory_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE) + return nil } _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr { - if alignment <= PAGE_SIZE { + if alignment <= page_size { // This is the simplest case. // // By virtue of binary arithmetic, any address aligned to a power of @@ -47,7 +65,7 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in if !ok { return nil } - assert_contextless(mmap_result % PAGE_SIZE == 0) + assert_contextless(mmap_result % uintptr(page_size) == 0) modulo := mmap_result & uintptr(alignment-1) if modulo != 0 { // The address is misaligned, so we must return an adjusted address @@ -58,24 +76,24 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in // Sanity-checking: // - The adjusted address is still page-aligned, so it is a valid argument for munmap. // - The adjusted address is aligned to the user's needs. - assert_contextless(adjusted_result % PAGE_SIZE == 0) + assert_contextless(adjusted_result % uintptr(page_size) == 0) assert_contextless(adjusted_result % uintptr(alignment) == 0) // Round the delta to a multiple of the page size. - delta = delta / PAGE_SIZE * PAGE_SIZE + delta = delta / uintptr(page_size) * uintptr(page_size) if delta > 0 { // Unmap the pages we don't need. intrinsics.syscall_bsd(SYS_munmap, mmap_result, delta) } return rawptr(adjusted_result) - } else if size + alignment > PAGE_SIZE { + } else if size + alignment > page_size { // The address is coincidentally aligned as desired, but we have space // that will never be seen by the user, so we must free the backing // pages for it. - start := size / PAGE_SIZE * PAGE_SIZE - if size % PAGE_SIZE != 0 { - start += PAGE_SIZE + start := size / page_size * page_size + if size % page_size != 0 { + start += page_size } length := size + alignment - start if length > 0 { diff --git a/base/runtime/virtual_memory_other.odin b/base/runtime/virtual_memory_other.odin index 2c6860510..a992c5022 100644 --- a/base/runtime/virtual_memory_other.odin +++ b/base/runtime/virtual_memory_other.odin @@ -9,6 +9,8 @@ package runtime VIRTUAL_MEMORY_SUPPORTED :: false +_init_virtual_memory :: proc "contextless" () { } + _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { unimplemented_contextless("Virtual memory is not supported on this platform.") } diff --git a/base/runtime/virtual_memory_windows.odin b/base/runtime/virtual_memory_windows.odin index cd39a3170..cf9ce5bd1 100644 --- a/base/runtime/virtual_memory_windows.odin +++ b/base/runtime/virtual_memory_windows.odin @@ -81,11 +81,26 @@ MEM_RELEASE :: 0x00008000 PAGE_READWRITE :: 0x04 +_allocation_granularity: int + +_init_virtual_memory :: proc "contextless" () { + sys_info: SYSTEM_INFO + GetSystemInfo(&sys_info) + + page_size = int(sys_info.dwPageSize) + _allocation_granularity = int(sys_info.dwAllocationGranularity) +} + +_get_superpage_size :: proc "contextless" () -> int { + // TODO: Windows has support for Large Pages, but its usage requires privilege escalation. + return 0 +} + _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { // `Size` must be a multiple of the page size. rounded_size := size - if rounded_size % PAGE_SIZE != 0 { - rounded_size = (size / PAGE_SIZE + 1) * PAGE_SIZE + if rounded_size % page_size != 0 { + rounded_size = (size / page_size + 1) * page_size } result := VirtualAlloc(nil, uint(rounded_size), MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE) if result == nil { @@ -95,14 +110,11 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr { } _allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr { - // TODO: Windows has support for Large Pages, but its usage requires privilege escalation. - return _allocate_virtual_memory_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE) + return nil } _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr { - sys_info: SYSTEM_INFO - GetSystemInfo(&sys_info) - if alignment <= int(sys_info.dwAllocationGranularity) { + if alignment <= _allocation_granularity { // The alignment is less than or equal to the allocation granularity, // which means it will automatically be aligned and any request for // alignment less than the allocation granularity will result in @@ -120,8 +132,8 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in } // `Size` must be a multiple of the page size. rounded_size := size - if rounded_size % PAGE_SIZE != 0 { - rounded_size = (size / PAGE_SIZE + 1) * PAGE_SIZE + if rounded_size % page_size != 0 { + rounded_size = (size / page_size + 1) * page_size } result := VirtualAlloc2(nil, nil, uint(rounded_size), MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE, ¶m, 1) if result == nil { diff --git a/tests/heap_allocator/test_bench.odin b/tests/heap_allocator/test_bench.odin index d01698841..4557532fc 100644 --- a/tests/heap_allocator/test_bench.odin +++ b/tests/heap_allocator/test_bench.odin @@ -890,7 +890,7 @@ main :: proc() { if opt.vmem_tests { log.info("Testing virtual memory allocation ...") - time.sleep(1 * time.Second) + log.infof("base:runtime reports OS page size is %M and superpage size is %M", runtime.page_size, runtime.superpage_size) for size in 12.. 0 { log.debugf("Testing superpage allocation and alignment ...") v := runtime.allocate_virtual_memory_superpage() - expect(uintptr(v) % runtime.SUPERPAGE_SIZE == 0) + expect(uintptr(v) % uintptr(size) == 0) va := cast([^]u8)v - for i in 0..