mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-08-05 05:38:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support getting page sizes at runtime

Browse Source
This commit is contained in:
Feoramund
2025-05-10 17:52:33 -04:00
parent 97a08e5c15
commit 4b08e8cfff
11 changed files with 402 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
base/runtime/heap_allocator_implementation.odin
View File

@@ -133,6 +133,7 @@ ODIN_HEAP_SMALL_BIN_MAX :: #config(ODIN_HEAP_SMALL_BIN_MAX, 8 * Kilobyte) // [
// Constants
//
ODIN_HEAP_SEGMENT_SIZE :: 4 * Megabyte
ODIN_HEAP_MIN_BIN_SHIFT :: intrinsics.constant_log2(ODIN_HEAP_MIN_BIN_SIZE)
ODIN_HEAP_MAX_BIN_SHIFT :: intrinsics.constant_log2(ODIN_HEAP_MAX_BIN_SIZE)
ODIN_HEAP_BIN_RANKS :: 1 + ODIN_HEAP_MAX_BIN_SHIFT - ODIN_HEAP_MIN_BIN_SHIFT
@@ -197,7 +198,7 @@ Heap_Debug_Level :: enum {
Heap_Slab_Class :: enum {
Small, // Slabs are `ODIN_HEAP_SMALL_SLAB_SIZE` (64KiB) each.
Large, // One segment-wide (~2MiB) slab.
Large, // One segment-wide (platform-dependent size) slab.
Huge, // One slab for one allocation, sized specifically for the request.
}
@@ -221,7 +222,12 @@ Allocate a new Segment that may be used to store either Small or Large slabs.
@(require_results)
heap_allocate_segment :: #force_inline proc "contextless" () -> ^Heap_Segment {
when ODIN_HEAP_SEGMENT_SIZE_OVERRIDE == 0 {
return cast(^Heap_Segment)allocate_virtual_memory_superpage()
if superpage_size != 0 {
return cast(^Heap_Segment)allocate_virtual_memory_superpage()
} else {
// Use the default segment value.
return cast(^Heap_Segment)allocate_virtual_memory_aligned(ODIN_HEAP_SEGMENT_SIZE, ODIN_HEAP_SEGMENT_SIZE)
}
} else {
return cast(^Heap_Segment)allocate_virtual_memory_aligned(ODIN_HEAP_SEGMENT_SIZE_OVERRIDE, ODIN_HEAP_SEGMENT_SIZE_OVERRIDE)
}
@@ -233,8 +239,11 @@ Get the constant size for all segments. This size also dictates each segment's a
@(require_results)
heap_get_segment_size :: #force_inline proc "contextless" () -> int {
when ODIN_HEAP_SEGMENT_SIZE_OVERRIDE == 0 {
// TODO: Derive from the OS config.
return SUPERPAGE_SIZE
if size := superpage_size; size != 0 {
return size
} else {
return ODIN_HEAP_SEGMENT_SIZE
}
} else {
return ODIN_HEAP_SEGMENT_SIZE_OVERRIDE
}
@@ -426,8 +435,6 @@ The **Segment** is a single contiguous allocation from the operating system's
virtual memory subsystem, subdivided into Slabs. All metadata lives at the head
of the allocation.
On almost every platform, this structure will be 2MiB by default.
Depending on the operating system, addresses within the space occupied by the
Segment (and hence its allocations) may also have faster access times due to
leveraging properties of the Translation Lookaside Buffer.

61
base/runtime/os_specific_sysv.odin Normal file
View File

@@ -0,0 +1,61 @@
#+private
#+build linux, freebsd, netbsd, openbsd
package runtime
// See the System Five Application Binary Interface § 3.4.3 for more information.
// Figure 3.11: Auxiliary Vector Types
Auxiliary_Vector_Type :: enum i32 {
AT_NULL = 0, // ignored
AT_IGNORE = 1, // ignored
AT_EXECFD = 2, // a_val
AT_PHDR = 3, // a_ptr
AT_PHENT = 4, // a_val
AT_PHNUM = 5, // a_val
AT_PAGESZ = 6, // a_val
AT_BASE = 7, // a_ptr
AT_FLAGS = 8, // a_val
AT_ENTRY = 9, // a_ptr
AT_NOTELF = 10, // a_val
AT_UID = 11, // a_val
AT_EUID = 12, // a_val
AT_GID = 13, // a_val
AT_EGID = 14, // a_val
}
@(private="file")
c_long :: i32 when size_of(rawptr) == 4 else i64
// Figure 3.10: auxv_t Type Definition
auxv_t :: struct {
a_type: Auxiliary_Vector_Type,
using a_un: struct #raw_union {
a_val: c_long, // long
a_ptr: rawptr, // void*
a_fnc: proc "c" (), // void (*)()
},
}
auxv__: [^]auxv_t
// Mind the alphanumeric sorted naming of the files in `base:runtime`, as this
// init needs to run before the virtual memory init.
@(init)
init_auxv :: proc "contextless" () {
// This is similar to how we get the environment on Linux.
#no_bounds_check auxv := cast([^]rawptr)&args__[len(args__) + 1]
for auxv[0] != nil {
auxv = auxv[1:]
}
auxv__ = cast([^]auxv_t)(auxv[1:])
}
// Get a value from the auxiliary vector.
_get_auxiliary :: proc "contextless" (at: Auxiliary_Vector_Type) -> (value: auxv_t, found: bool) {
for ap := auxv__; ap != nil && ap[0].a_type != .AT_NULL; ap = ap[1:] {
if ap[0].a_type == at {
return ap[0], true
}
}
return
}

37
base/runtime/virtual_memory.odin
View File

@@ -1,21 +1,26 @@
package runtime
import "base:intrinsics"
ODIN_VIRTUAL_MEMORY_SUPPORTED :: VIRTUAL_MEMORY_SUPPORTED
// Virtually all MMUs supported by Odin should have a 4KiB page size.
PAGE_SIZE :: 4 * Kilobyte
/*
The page size of the operating system, used for virtual memory allocations.
*/
page_size: int
when ODIN_ARCH == .arm32 {
SUPERPAGE_SIZE :: 1 * Megabyte
} else {
// All other architectures should have support for 2MiB pages.
// i386 supports it in PAE mode.
// amd64, arm64, and riscv64 support it by default.
SUPERPAGE_SIZE :: 2 * Megabyte
/*
The superpage size of the operating system.
This may be zero if unavailable.
*/
superpage_size: int
@(init, private)
init_virtual_memory :: proc "contextless" () {
_init_virtual_memory()
}
#assert(SUPERPAGE_SIZE & (SUPERPAGE_SIZE-1) == 0, "SUPERPAGE_SIZE must be a power of two.")
/*
Allocate virtual memory from the operating system.
@@ -35,14 +40,20 @@ This is a contiguous block of memory larger than what is normally distributed
by the operating system, sometimes with special performance properties related
to the Translation Lookaside Buffer.
The address will be a multiple of the `SUPERPAGE_SIZE` constant, and the memory
pointed to will be at least as long as that very same constant.
The address will be a multiple of `superpage_size`, and the memory
pointed to will be at least as long as that.
The name derives from the superpage concept on the *BSD operating systems,
where it is known as huge pages on Linux and large pages on Windows.
This may return nil if a superpage size was unable to be retrieved from the
operating system or if the feature is otherwise unavailable.
*/
@(require_results)
allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr {
if superpage_size == 0 {
return nil
}
return _allocate_virtual_memory_superpage()
}

38
base/runtime/virtual_memory_darwin.odin
View File

@@ -8,6 +8,7 @@ VIRTUAL_MEMORY_SUPPORTED :: true
foreign import lib "system:System.framework"
foreign lib {
vm_page_size: uintptr
mach_task_self_: u32
mach_vm_allocate :: proc(target: u32, address: ^u64, size: u64, flags: i32) -> i32 ---
mach_vm_deallocate :: proc(target: u32, address: u64, size: u64) -> i32 ---
@@ -55,6 +56,25 @@ VM_PROT_READ :: 0x01
VM_PROT_WRITE :: 0x02
VM_INHERIT_COPY :: 1
_init_virtual_memory :: proc "contextless" () {
page_size = _get_page_size()
superpage_size = _get_superpage_size()
}
_get_page_size :: proc "contextless" () -> int {
return int(vm_page_size)
}
_get_superpage_size :: proc "contextless" () -> int {
when ODIN_ARCH == .amd64 {
// NOTE(Feoramund): As far as we are aware, Darwin only supports
// explicit superpage allocation on AMD64 with a 2MiB parameter.
return 2 * Megabyte
} else {
return 0
}
}
_allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
address: u64
result := mach_vm_map(mach_task_self_, &address, u64(size), 0, VM_FLAGS_ANYWHERE, MEMORY_OBJECT_NULL, 0, false, VM_PROT_READ|VM_PROT_WRITE, VM_PROT_READ|VM_PROT_WRITE, VM_INHERIT_COPY)
@@ -66,22 +86,14 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
_allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr {
address: u64
flags: i32 = VM_FLAGS_ANYWHERE
when ODIN_ARCH == .amd64 {
// NOTE(Feoramund): As far as we are aware, Darwin only supports
// explicit superpage allocation on AMD64 with a 2MiB parameter.
when SUPERPAGE_SIZE == 2 * Megabyte {
flags |= VM_FLAGS_SUPERPAGE_SIZE_2MB
} else {
#panic("An unsupported superpage size has been configured for AMD64 Darwin; only 2MB is supported.")
}
}
alignment_mask: u64 = SUPERPAGE_SIZE - 1 // Assumes a power of two size, ensured by an assertion in `virtual_memory.odin`.
result := mach_vm_map(mach_task_self_, &address, SUPERPAGE_SIZE, alignment_mask, flags, MEMORY_OBJECT_NULL, 0, false, VM_PROT_READ|VM_PROT_WRITE, VM_PROT_READ|VM_PROT_WRITE, VM_INHERIT_COPY)
flags: i32 = VM_FLAGS_ANYWHERE | VM_FLAGS_SUPERPAGE_SIZE_2MB
assert_contextless(superpage_size & (superpage_size-1) == 0, "The superpage size is not a power of two.")
alignment_mask: u64 = u64(superpage_size) - 1
result := mach_vm_map(mach_task_self_, &address, 2 * Megabyte, alignment_mask, flags, MEMORY_OBJECT_NULL, 0, false, VM_PROT_READ|VM_PROT_WRITE, VM_PROT_READ|VM_PROT_WRITE, VM_INHERIT_COPY)
if result != 0 {
return nil
}
assert_contextless(address % SUPERPAGE_SIZE == 0)
assert_contextless(address % u64(superpage_size) == 0)
return rawptr(uintptr(address))
}

65
base/runtime/virtual_memory_freebsd.odin
View File

@@ -25,7 +25,46 @@ MAP_ANONYMOUS :: 0x1000
MAP_ALIGNMENT_SHIFT :: 24
MAP_ALIGNED_SUPER :: 1 << MAP_ALIGNMENT_SHIFT
SUPERPAGE_MAP_FLAGS :: (intrinsics.constant_log2(SUPERPAGE_SIZE) << MAP_ALIGNMENT_SHIFT) | MAP_ALIGNED_SUPER
_init_virtual_memory :: proc "contextless" () {
page_size = _get_page_size()
superpage_size = _get_superpage_size()
}
_get_page_size :: proc "contextless" () -> int {
// This is a fallback value if the auxiliary vector does not supply it.
DEFAULT_PAGE_SIZE :: 4096
if value, found := _get_auxiliary(.AT_PAGESZ); found {
return int(value.a_val)
} else {
return DEFAULT_PAGE_SIZE
}
}
_get_superpage_size :: proc "contextless" () -> int {
// This is specific to FreeBSD and not defined in the SysV ABI.
AT_PAGESIZES :: Auxiliary_Vector_Type(20)
if value, found := _get_auxiliary(AT_PAGESIZES); found {
greatest_size := 0
supports_2mib := false
for sizes := cast([^]uint)value.a_ptr; sizes[0] != 0; sizes = sizes[1:] {
if sizes[0] == 2 * Megabyte {
// The standard 2MiB superpage is supported.
supports_2mib = true
}
greatest_size = max(greatest_size, int(sizes[0]))
}
if supports_2mib {
return 2 * Megabyte
} else if greatest_size > _get_page_size() {
return greatest_size
}
}
return 0
}
_allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0)
@@ -36,11 +75,13 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
}
_allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr {
result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, SUPERPAGE_SIZE, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|SUPERPAGE_MAP_FLAGS, ~uintptr(0), 0)
superpage_flags := uintptr(intrinsics.count_trailing_zeros(superpage_size) << MAP_ALIGNMENT_SHIFT) | MAP_ALIGNED_SUPER
result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(superpage_size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|superpage_flags, ~uintptr(0), 0)
if !ok {
// It may be the case that FreeBSD couldn't fulfill our alignment
// request, but it could still give us some memory.
return _allocate_virtual_memory_manually_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE)
return _allocate_virtual_memory_manually_aligned(superpage_size, superpage_size)
}
return rawptr(result)
}
@@ -49,7 +90,7 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in
// This procedure uses the `MAP_ALIGNED` API provided by FreeBSD and falls
// back to manually aligned addresses, if that fails.
map_aligned_n: uintptr
if alignment >= PAGE_SIZE {
if alignment >= page_size {
map_aligned_n = intrinsics.count_trailing_zeros(uintptr(alignment)) << MAP_ALIGNMENT_SHIFT
}
result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|map_aligned_n, ~uintptr(0), 0)
@@ -60,7 +101,7 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in
}
_allocate_virtual_memory_manually_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr {
if alignment <= PAGE_SIZE {
if alignment <= page_size {
// This is the simplest case.
//
// By virtue of binary arithmetic, any address aligned to a power of
@@ -78,7 +119,7 @@ _allocate_virtual_memory_manually_aligned :: proc "contextless" (size: int, alig
if !ok {
return nil
}
assert_contextless(mmap_result % PAGE_SIZE == 0)
assert_contextless(mmap_result % uintptr(page_size) == 0)
modulo := mmap_result & uintptr(alignment-1)
if modulo != 0 {
// The address is misaligned, so we must return an adjusted address
@@ -89,24 +130,24 @@ _allocate_virtual_memory_manually_aligned :: proc "contextless" (size: int, alig
// Sanity-checking:
// - The adjusted address is still page-aligned, so it is a valid argument for munmap.
// - The adjusted address is aligned to the user's needs.
assert_contextless(adjusted_result % PAGE_SIZE == 0)
assert_contextless(adjusted_result % uintptr(page_size) == 0)
assert_contextless(adjusted_result % uintptr(alignment) == 0)
// Round the delta to a multiple of the page size.
delta = delta / PAGE_SIZE * PAGE_SIZE
delta = delta / uintptr(page_size) * uintptr(page_size)
if delta > 0 {
// Unmap the pages we don't need.
intrinsics.syscall_bsd(SYS_munmap, mmap_result, delta)
}
return rawptr(adjusted_result)
} else if size + alignment > PAGE_SIZE {
} else if size + alignment > page_size {
// The address is coincidentally aligned as desired, but we have space
// that will never be seen by the user, so we must free the backing
// pages for it.
start := size / PAGE_SIZE * PAGE_SIZE
if size % PAGE_SIZE != 0 {
start += PAGE_SIZE
start := size / page_size * page_size
if size % page_size != 0 {
start += page_size
}
length := size + alignment - start
if length > 0 {

173
base/runtime/virtual_memory_linux.odin
View File

@@ -6,25 +6,45 @@ import "base:intrinsics"
VIRTUAL_MEMORY_SUPPORTED :: true
when ODIN_ARCH == .amd64 {
SYS_mmap :: uintptr(9)
SYS_munmap :: uintptr(11)
SYS_mremap :: uintptr(25)
SYS_open :: uintptr(2)
SYS_read :: uintptr(0)
SYS_close :: uintptr(3)
SYS_mmap :: uintptr(9)
SYS_munmap :: uintptr(11)
SYS_mremap :: uintptr(25)
} else when ODIN_ARCH == .arm32 {
SYS_mmap :: uintptr(90)
SYS_munmap :: uintptr(91)
SYS_mremap :: uintptr(163)
SYS_open :: uintptr(5)
SYS_read :: uintptr(3)
SYS_close :: uintptr(6)
SYS_mmap :: uintptr(90)
SYS_munmap :: uintptr(91)
SYS_mremap :: uintptr(163)
} else when ODIN_ARCH == .arm64 {
SYS_mmap :: uintptr(222)
SYS_munmap :: uintptr(215)
SYS_mremap :: uintptr(216)
SYS_openat :: uintptr(56)
SYS_read :: uintptr(63)
SYS_close :: uintptr(57)
SYS_mmap :: uintptr(222)
SYS_munmap :: uintptr(215)
SYS_mremap :: uintptr(216)
} else when ODIN_ARCH == .i386 {
SYS_mmap :: uintptr(90)
SYS_munmap :: uintptr(91)
SYS_mremap :: uintptr(163)
SYS_open :: uintptr(5)
SYS_read :: uintptr(3)
SYS_close :: uintptr(6)
SYS_mmap :: uintptr(90)
SYS_munmap :: uintptr(91)
SYS_mremap :: uintptr(163)
} else when ODIN_ARCH == .riscv64 {
SYS_mmap :: uintptr(222)
SYS_munmap :: uintptr(215)
SYS_mremap :: uintptr(216)
SYS_openat :: uintptr(56)
SYS_read :: uintptr(63)
SYS_close :: uintptr(57)
SYS_mmap :: uintptr(222)
SYS_munmap :: uintptr(215)
SYS_mremap :: uintptr(216)
} else {
#panic("Syscall numbers related to virtual memory are missing for this Linux architecture.")
}
@@ -39,6 +59,105 @@ MREMAP_MAYMOVE :: 0x01
ENOMEM :: ~uintptr(11)
_init_virtual_memory :: proc "contextless" () {
page_size = _get_page_size()
superpage_size = _get_superpage_size()
}
_get_page_size :: proc "contextless" () -> int {
// This is a fallback value if the auxiliary vector does not supply it.
DEFAULT_PAGE_SIZE :: 4096
if value, found := _get_auxiliary(.AT_PAGESZ); found {
return int(value.a_val)
} else {
return DEFAULT_PAGE_SIZE
}
}
_get_superpage_size :: proc "contextless" () -> int {
meminfo: cstring = "/proc/meminfo"
when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 {
AT_FDCWD :: ~uintptr(99) // -100
fd := cast(int)intrinsics.syscall(SYS_openat, AT_FDCWD, transmute(uintptr)meminfo, 0 /* flags */, 0 /* mode */)
} else {
fd := cast(int)intrinsics.syscall(SYS_open, transmute(uintptr)meminfo, 0 /* flags */, 0 /* mode */)
}
if fd < 0 {
// Error on opening file.
return 0
}
defer intrinsics.syscall(SYS_close, uintptr(fd))
buf: [4096]u8
read := cast(int)intrinsics.syscall(SYS_read, cast(uintptr)fd, cast(uintptr)&buf[0], len(buf))
if read <= 0 {
// Failed to read anything.
return 0
}
// Parse the file. It's in a format of "KEY: VALUE\n" with a
// variable number of spaces after the colon.
str := buf[:read]
for len(str) > 0 {
key, val: []u8
// Get the key.
for c, i in str {
if c == ':' {
key, str = str[:i], str[1+i:]
break
}
}
// Trim the spaces.
for c, i in str {
if c != ' ' {
str = str[i:]
break
}
}
// Get the value.
for c, i in str {
if c == '\n' {
val, str = str[:i], str[1+i:]
break
}
}
// Break in the event something was parsed incorrectly.
if len(key) == 0 || len(val) == 0 {
break
}
if string(key) == "Hugepagesize" {
// The value will be in a format like: 2048 kB
n, unit: []u8
for c, i in val {
if c == ' ' {
n = val[:i]
unit = val[1+i:]
break
}
}
// Convert it to a number.
bytes := 0
for c in n {
bytes *= 10
bytes += int(c - '0')
}
// The man page for `proc_meminfo` does not state if it
// uses measurements other than "kB" but just to be safe.
switch string(unit) {
case "kB": bytes *= Kilobyte
case "mB": bytes *= Megabyte
case "gB": bytes *= Gigabyte
}
return bytes
}
}
return 0
}
_allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
result := intrinsics.syscall(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0)
if int(result) < 0 {
@@ -49,22 +168,22 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
_allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr {
// This depends on Transparent HugePage Support being enabled.
result := intrinsics.syscall(SYS_mmap, 0, SUPERPAGE_SIZE, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0)
result := intrinsics.syscall(SYS_mmap, 0, uintptr(superpage_size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0)
if int(result) < 0 {
return nil
}
if uintptr(result) % SUPERPAGE_SIZE != 0 {
if uintptr(result) % uintptr(superpage_size) != 0 {
// If THP support is not enabled, we may receive an address aligned to a
// page boundary instead, in which case, we must manually align a new
// address.
_free_virtual_memory(rawptr(result), SUPERPAGE_SIZE)
return _allocate_virtual_memory_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE)
_free_virtual_memory(rawptr(result), superpage_size)
return _allocate_virtual_memory_aligned(superpage_size, superpage_size)
}
return rawptr(result)
}
_allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr {
if alignment <= PAGE_SIZE {
if alignment <= page_size {
// This is the simplest case.
//
// By virtue of binary arithmetic, any address aligned to a power of
@@ -82,7 +201,7 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in
if int(mmap_result) < 0 {
return nil
}
assert_contextless(mmap_result % PAGE_SIZE == 0)
assert_contextless(mmap_result % uintptr(page_size) == 0)
modulo := mmap_result & uintptr(alignment-1)
if modulo != 0 {
// The address is misaligned, so we must return an adjusted address
@@ -93,24 +212,24 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in
// Sanity-checking:
// - The adjusted address is still page-aligned, so it is a valid argument for mremap and munmap.
// - The adjusted address is aligned to the user's needs.
assert_contextless(adjusted_result % PAGE_SIZE == 0)
assert_contextless(adjusted_result % uintptr(page_size) == 0)
assert_contextless(adjusted_result % uintptr(alignment) == 0)
// Round the delta to a multiple of the page size.
delta = delta / PAGE_SIZE * PAGE_SIZE
delta = delta / uintptr(page_size) * uintptr(page_size)
if delta > 0 {
// Unmap the pages we don't need.
intrinsics.syscall(SYS_munmap, mmap_result, delta)
}
return rawptr(adjusted_result)
} else if size + alignment > PAGE_SIZE {
} else if size + alignment > page_size {
// The address is coincidentally aligned as desired, but we have space
// that will never be seen by the user, so we must free the backing
// pages for it.
start := size / PAGE_SIZE * PAGE_SIZE
if size % PAGE_SIZE != 0 {
start += PAGE_SIZE
start := size / page_size * page_size
if size % page_size != 0 {
start += page_size
}
length := size + alignment - start
if length > 0 {

24
base/runtime/virtual_memory_netbsd.odin
View File

@@ -23,6 +23,26 @@ MAP_ANONYMOUS :: 0x1000
// #define MAP_ALIGNED(n) ((int)((unsigned int)(n) << MAP_ALIGNMENT_SHIFT))
MAP_ALIGNMENT_SHIFT :: 24
_init_virtual_memory :: proc "contextless" () {
page_size = _get_page_size()
}
_get_page_size :: proc "contextless" () -> int {
// This is a fallback value if the auxiliary vector does not supply it.
DEFAULT_PAGE_SIZE :: 4096
if value, found := _get_auxiliary(.AT_PAGESZ); found {
return int(value.a_val)
} else {
return DEFAULT_PAGE_SIZE
}
}
_get_superpage_size :: proc "contextless" () -> int {
// NOTE(Feoramund): I am uncertain if NetBSD has direct support for superpages.
return 0
}
_allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0)
if !ok {
@@ -32,9 +52,7 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
}
_allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr {
// NOTE(Feoramund): I am uncertain if NetBSD has direct support for
// superpages, so we just use the aligned allocate procedure here.
return _allocate_virtual_memory_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE)
return nil
}
_allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr {

40
base/runtime/virtual_memory_openbsd.odin
View File

@@ -14,6 +14,26 @@ PROT_WRITE :: 0x02
MAP_PRIVATE :: 0x0002
MAP_ANONYMOUS :: 0x1000
_init_virtual_memory :: proc "contextless" () {
page_size = _get_page_size()
}
_get_page_size :: proc "contextless" () -> int {
// This is a fallback value if the auxiliary vector does not supply it.
DEFAULT_PAGE_SIZE :: 4096
if value, found := _get_auxiliary(.AT_PAGESZ); found {
return int(value.a_val)
} else {
return DEFAULT_PAGE_SIZE
}
}
_get_superpage_size :: proc "contextless" () -> int {
// NOTE(Feoramund): I am uncertain if OpenBSD has direct support for superpages.
return 0
}
_allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
result, ok := intrinsics.syscall_bsd(SYS_mmap, 0, uintptr(size), PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, ~uintptr(0), 0)
if !ok {
@@ -23,13 +43,11 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
}
_allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr {
// NOTE(Feoramund): I am uncertain if OpenBSD has direct support for
// superpages, so we just use the aligned allocate procedure here.
return _allocate_virtual_memory_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE)
return nil
}
_allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr {
if alignment <= PAGE_SIZE {
if alignment <= page_size {
// This is the simplest case.
//
// By virtue of binary arithmetic, any address aligned to a power of
@@ -47,7 +65,7 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in
if !ok {
return nil
}
assert_contextless(mmap_result % PAGE_SIZE == 0)
assert_contextless(mmap_result % uintptr(page_size) == 0)
modulo := mmap_result & uintptr(alignment-1)
if modulo != 0 {
// The address is misaligned, so we must return an adjusted address
@@ -58,24 +76,24 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in
// Sanity-checking:
// - The adjusted address is still page-aligned, so it is a valid argument for munmap.
// - The adjusted address is aligned to the user's needs.
assert_contextless(adjusted_result % PAGE_SIZE == 0)
assert_contextless(adjusted_result % uintptr(page_size) == 0)
assert_contextless(adjusted_result % uintptr(alignment) == 0)
// Round the delta to a multiple of the page size.
delta = delta / PAGE_SIZE * PAGE_SIZE
delta = delta / uintptr(page_size) * uintptr(page_size)
if delta > 0 {
// Unmap the pages we don't need.
intrinsics.syscall_bsd(SYS_munmap, mmap_result, delta)
}
return rawptr(adjusted_result)
} else if size + alignment > PAGE_SIZE {
} else if size + alignment > page_size {
// The address is coincidentally aligned as desired, but we have space
// that will never be seen by the user, so we must free the backing
// pages for it.
start := size / PAGE_SIZE * PAGE_SIZE
if size % PAGE_SIZE != 0 {
start += PAGE_SIZE
start := size / page_size * page_size
if size % page_size != 0 {
start += page_size
}
length := size + alignment - start
if length > 0 {

2
base/runtime/virtual_memory_other.odin
View File

@@ -9,6 +9,8 @@ package runtime
VIRTUAL_MEMORY_SUPPORTED :: false
_init_virtual_memory :: proc "contextless" () { }
_allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
unimplemented_contextless("Virtual memory is not supported on this platform.")
}

30
base/runtime/virtual_memory_windows.odin
View File

@@ -81,11 +81,26 @@ MEM_RELEASE :: 0x00008000
PAGE_READWRITE :: 0x04
_allocation_granularity: int
_init_virtual_memory :: proc "contextless" () {
sys_info: SYSTEM_INFO
GetSystemInfo(&sys_info)
page_size = int(sys_info.dwPageSize)
_allocation_granularity = int(sys_info.dwAllocationGranularity)
}
_get_superpage_size :: proc "contextless" () -> int {
// TODO: Windows has support for Large Pages, but its usage requires privilege escalation.
return 0
}
_allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
// `Size` must be a multiple of the page size.
rounded_size := size
if rounded_size % PAGE_SIZE != 0 {
rounded_size = (size / PAGE_SIZE + 1) * PAGE_SIZE
if rounded_size % page_size != 0 {
rounded_size = (size / page_size + 1) * page_size
}
result := VirtualAlloc(nil, uint(rounded_size), MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE)
if result == nil {
@@ -95,14 +110,11 @@ _allocate_virtual_memory :: proc "contextless" (size: int) -> rawptr {
}
_allocate_virtual_memory_superpage :: proc "contextless" () -> rawptr {
// TODO: Windows has support for Large Pages, but its usage requires privilege escalation.
return _allocate_virtual_memory_aligned(SUPERPAGE_SIZE, SUPERPAGE_SIZE)
return nil
}
_allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: int) -> rawptr {
sys_info: SYSTEM_INFO
GetSystemInfo(&sys_info)
if alignment <= int(sys_info.dwAllocationGranularity) {
if alignment <= _allocation_granularity {
// The alignment is less than or equal to the allocation granularity,
// which means it will automatically be aligned and any request for
// alignment less than the allocation granularity will result in
@@ -120,8 +132,8 @@ _allocate_virtual_memory_aligned :: proc "contextless" (size: int, alignment: in
}
// `Size` must be a multiple of the page size.
rounded_size := size
if rounded_size % PAGE_SIZE != 0 {
rounded_size = (size / PAGE_SIZE + 1) * PAGE_SIZE
if rounded_size % page_size != 0 {
rounded_size = (size / page_size + 1) * page_size
}
result := VirtualAlloc2(nil, nil, uint(rounded_size), MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE, &param, 1)
if result == nil {

18
tests/heap_allocator/test_bench.odin
View File

@@ -890,7 +890,7 @@ main :: proc() {
if opt.vmem_tests {
log.info("Testing virtual memory allocation ...")
time.sleep(1 * time.Second)
log.infof("base:runtime reports OS page size is %M and superpage size is %M", runtime.page_size, runtime.superpage_size)
for size in 12..<uint(22) {
size := 1 << size
for shift in 0..<uint(22) {
@@ -917,24 +917,23 @@ main :: proc() {
runtime.free_virtual_memory(v, size+1)
}
}
{
if size := runtime.superpage_size; size > 0 {
log.debugf("Testing superpage allocation and alignment ...")
v := runtime.allocate_virtual_memory_superpage()
expect(uintptr(v) % runtime.SUPERPAGE_SIZE == 0)
expect(uintptr(v) % uintptr(size) == 0)
va := cast([^]u8)v
for i in 0..<runtime.SUPERPAGE_SIZE {
for i in 0..<size {
expect(va[i] == 0)
}
for i in 0..<runtime.SUPERPAGE_SIZE {
for i in 0..<size {
va[i] = 0xAA
}
for i in 0..<runtime.SUPERPAGE_SIZE {
for i in 0..<size {
expect(va[i] == 0xAA)
}
runtime.free_virtual_memory(v, runtime.SUPERPAGE_SIZE)
runtime.free_virtual_memory(v, size)
}
log.info("Done.")
time.sleep(3 * time.Second)
}
if opt.parallel_tests {
@@ -1090,7 +1089,6 @@ main :: proc() {
bench_alloc_n_then_free_n(100_000, [4096*4]u8)
bench_alloc_n_then_free_n(10_000, [65536/4]u8)
bench_alloc_n_then_free_n(10_000, [65536*4]u8)
bench_alloc_n_then_free_n(100, [runtime.SUPERPAGE_SIZE]u8)
log.info("* Freeing backwards ...")
bench_alloc_n_then_free_n_backwards(10_000_000, int)
@@ -1101,7 +1099,6 @@ main :: proc() {
bench_alloc_n_then_free_n_backwards(100_000, [8192]u8)
bench_alloc_n_then_free_n_backwards(10_000, [65536/4]u8)
bench_alloc_n_then_free_n_backwards(10_000, [65536*4]u8)
bench_alloc_n_then_free_n_backwards(100, [runtime.SUPERPAGE_SIZE]u8)
log.info("* Freeing randomly ...")
bench_alloc_n_then_free_n_randomly(10_000_000, int)
@@ -1113,7 +1110,6 @@ main :: proc() {
bench_alloc_n_then_free_n_randomly(100_000, [65536/4]u8)
bench_alloc_n_then_free_n_randomly(100_000, [65536]u8)
bench_alloc_n_then_free_n_randomly(100_000, [65536*2]u8)
bench_alloc_n_then_free_n_randomly(100, [runtime.SUPERPAGE_SIZE]u8)
log.info("* Allocating and freeing repeatedly ...")
bench_alloc_1_then_free_1_repeatedly(100_000, int)