From 439edc5fa8e5033b4881dceaedf73b4b1be00e95 Mon Sep 17 00:00:00 2001 From: corley <28909106+corleypc@users.noreply.github.com> Date: Mon, 3 Aug 2026 02:32:09 +0300 Subject: [PATCH] make soa builtins go vroom --- base/runtime/core_builtin_soa.odin | 478 ++++++++++++++++------ tests/core/runtime/test_core_runtime.odin | 193 +++++++++ tests/core/speed.odin | 3 +- 3 files changed, 556 insertions(+), 118 deletions(-) diff --git a/base/runtime/core_builtin_soa.odin b/base/runtime/core_builtin_soa.odin index f4cdec08f..9dca88d9f 100644 --- a/base/runtime/core_builtin_soa.odin +++ b/base/runtime/core_builtin_soa.odin @@ -50,13 +50,18 @@ Raw_SOA_Footer_Dynamic_Array :: struct { allocator: Allocator, } +// Note: When casting array to access footer/fields +// uintptr(array) lowers to LLVM ptrtoint and captures pointer provenance, +// whcih defeats #no_alias on the array in any code following (including in callers). +// Multipointer indexing lowers to GEP and doesn't capture, so prefer that throughout. + @(builtin, require_results) raw_soa_footer_slice :: proc(array: ^$T/#soa[]$E) -> (footer: ^Raw_SOA_Footer_Slice) { if array == nil { return nil } - field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)) - footer = (^Raw_SOA_Footer_Slice)(uintptr(array) + field_count*size_of(rawptr)) + field_count := len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E) + footer = (^Raw_SOA_Footer_Slice)(&([^]byte)(array)[field_count*size_of(rawptr)]) return } @(builtin, require_results) @@ -64,8 +69,8 @@ raw_soa_footer_dynamic_array :: proc(array: ^$T/#soa[dynamic]$E) -> (footer: ^Ra if array == nil { return nil } - field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)) - footer = (^Raw_SOA_Footer_Dynamic_Array)(uintptr(array) + field_count*size_of(rawptr)) + field_count := len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E) + footer = (^Raw_SOA_Footer_Dynamic_Array)(&([^]byte)(array)[field_count*size_of(rawptr)]) return } raw_soa_footer :: proc{ @@ -118,15 +123,13 @@ make_soa_aligned :: proc($T: typeid/#soa[]$E, #any_int length, alignment: int, a } new_data := raw_data(new_bytes) - data := uintptr(&array) offset := 0 for i in 0.. Allocator_Error { +resize_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #caller_location) -> Allocator_Error { if array == nil { return nil } @@ -215,7 +218,7 @@ resize_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #cal } @builtin -non_zero_resize_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #caller_location) -> Allocator_Error { +non_zero_resize_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #caller_location) -> Allocator_Error { if array == nil { return nil } @@ -225,6 +228,15 @@ non_zero_resize_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int length: int, lo return nil } +// `reserve_soa` will try to reserve memory of a passed SOA dynamic array to the requested element count (setting the `cap`). +// +// For maximizing performance it is recommended to avoid power-of-2 capacities +// when manually reserving memory for SOA arrays, more so for sizes above 512. +// For element types with several fields, prefer padding such capacities by at +// least the number of fields that fit in a CPU cache line. +// Modern cache lines are typically 64 (Intel/AMD/Arm64) or 128 bytes (Apple Silicon), +// so pad requested capacity with e.g. `128 / size_of(field)` (for the smallest field when field sizes differ), +// that is, prefer capacity of 4128 instead of 4096 for 4-byte fields. @builtin reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int capacity: int, loc := #caller_location) -> Allocator_Error { return _reserve_soa(array, capacity, true, loc) @@ -235,7 +247,33 @@ non_zero_reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int capacity: int, return _reserve_soa(array, capacity, false, loc) } -_reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, capacity: int, zero_memory: bool, loc := #caller_location) -> Allocator_Error { + +// Note: capacity and performance +// Each field is stored in contiguous memory with stride between fields capacity * size_of(field) (+ alignment). +// When the stride is a multiple of 4KB, e.g. a power-of-2 capacity >= 1024 with 4-byte fields, +// every same-size field's cache line for a given index maps to the same L1 set, and +// per-element operations degrade (e.g. append_soa, inject_at_soa). +// It is worst past 8 same-size fields where the burst exceeds the common +// L1 8-way associativity and the fields collide (measured up to ~7-8x degradation +// on a type with 18 same-size fields; fields of different sizes advance through +// the sets at different rates, so they don't collide persistently). +// Common SoA iterative flows may also be affected, when the loop touches multiple fields +// and other hot data aliases the same L1 set. +// So, for types with several fields, prefer padding such capacities by at +// least the number of fields that fit in a CPU cache line. +// Modern cache lines are typically 64 (Intel/AMD/Arm64) or 128 bytes (Apple Silicon), +// so pad with e.g. 128 / size_of(field) (for the smallest field when field sizes differ), +// so prefer 4128 instead of 4096 for 4-byte fields. Setting cap to e.g. 4097 is NOT enough +// (it shifts field memory by only 4 bytes (same cache line) and fields keep colliding). +// Note: 4KB stride in the discussion above is the x86 L1 set period (32–48KB/8–12-way); +// Apple Silicon (128KB/8-way) has a 16KB period, so cap 4096 × 4 = 16KB stride +// is still pathological there and the recommended 128-byte pad fixes it. +// Note: struct size also plays a role, but the user facing padding recommendation +// on reserve_soa() will handle up to 256 fields (of 4 bytes) decently on 8-way caches. +// Note: Ideally, this should be handled internally by allowing allocated capacity to exceed requested capacity. +// This would break current runtime tests though, they explicitly test requested cap. +// Note: Capacities produced by append growth all the way from empty (2*cap + 8) are not affected. +_reserve_soa :: #force_no_inline proc(array: ^$T/#soa[dynamic]$E, capacity: int, zero_memory: bool, loc := #caller_location) -> Allocator_Error { if array == nil { return nil } @@ -322,7 +360,7 @@ _reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, capacity: int, zero_memory: boo mem_copy(new_data_elem, old_data_elem, old_size_elem) - (^rawptr)(uintptr(array) + i*size_of(rawptr))^ = new_data_elem + ([^]rawptr)(array)[i] = new_data_elem if zero_memory { mem_zero(rawptr(uintptr(new_data_elem) + uintptr(old_size_elem)), new_size_elem - old_size_elem) @@ -361,7 +399,7 @@ _reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, capacity: int, zero_memory: boo mem_copy(new_data_elem, old_data_elem, type.size * old_cap) - (^rawptr)(uintptr(array) + i*size_of(rawptr))^ = new_data_elem + ([^]rawptr)(array)[i] = new_data_elem old_offset += type.size * old_cap new_offset += type.size * capacity @@ -388,7 +426,7 @@ non_zero_append_soa_elem :: proc(array: ^$T/#soa[dynamic]$E, #no_broadcast arg: return _append_soa_elem(array, false, arg, loc) } -_append_soa_elem :: proc(array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broadcast arg: E, loc := #caller_location) -> (n: int, err: Allocator_Error) #optional_allocator_error { +_append_soa_elem :: proc(#no_alias array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broadcast arg: E, loc := #caller_location) -> (n: int, err: Allocator_Error) #optional_allocator_error { if array == nil { return 0, nil } @@ -399,35 +437,13 @@ _append_soa_elem :: proc(array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broa err = _reserve_soa(array, cap, zero_memory, loc) // do not 'or_return' here as it could be a partial success } - footer := raw_soa_footer(array) - if size_of(E) > 0 && cap(array)-len(array) > 0 { - ti := type_info_of(T) - ti = type_info_base(ti) - si := &ti.variant.(Type_Info_Struct) - field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)) - - data := (^rawptr)(array)^ - - soa_offset := 0 - item_offset := 0 - - arg_copy := arg - arg_ptr := &arg_copy - - max_align :: align_of(E) - for i in 0.. (n: int, err: Allocator_Error) #optional_allocator_error { +_append_soa_elems :: proc(#no_alias array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broadcast args: []E, loc := #caller_location) -> (n: int, err: Allocator_Error) #optional_allocator_error { if array == nil { return } @@ -464,41 +480,267 @@ _append_soa_elems :: proc(array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_bro footer := raw_soa_footer(array) if size_of(E) > 0 && arg_len > 0 { - ti := type_info_of(typeid_of(T)) - ti = type_info_base(ti) - si := &ti.variant.(Type_Info_Struct) - field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)) - - data := (^rawptr)(array)^ - - soa_offset := 0 - item_offset := 0 - - args_ptr := &args[0] - - max_align :: align_of(E) - for i in 0..= .Speed do per-field copy passes that bind + // the SOA struct's multipointers and each incoming element's fields by + // position via expand_values with compile-time known types. + // At ODIN_OPTIMIZATION_MODE <= .Size the compiler's lowering is + // used, because it is the most compact at these field counts. + // When E has >16 fields: + // Type erased per-field loop using RTTI, with one mem_copy per element per field (codegen size is field count invariant). + // (Note that offset and the multipointers must be read after _reserve_soa, because any growth moves them.) + offset := footer.len + FIELD_COUNT :: len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E) + when FIELD_COUNT <= 16 { + when ODIN_OPTIMIZATION_MODE <= .Size { + // Use the compiler's #soa element store lowering. + // Note that #no_bounds_check is not optional, we write at index == len. + #no_bounds_check { + for j in 0..= 1 { // nothing to do if field count is 0 + _append_soa_elems_per_field_expanded(array, offset, args[:arg_len]) } + } else { // FIELD_COUNT > 16 + ti := type_info_base(type_info_of(typeid_of(T))) + si := &ti.variant.(Type_Info_Struct) + // si describes the SOA struct (fields are multipointers), so E's + // field offsets are not available in it; read them from E's own RTTI (si.soa_base_type). + // Note that basing on default struct layout here instead + // would misplace the fields of #packed elements. + // (> 16 fields is struct-only: array element types are capped at length 4.) + se := &type_info_base(si.soa_base_type).variant.(Type_Info_Struct) - soa_offset += type.size * cap(array) - item_offset += type.size + src_base := uintptr(raw_data(args)) + for i in 0.. (ok: bool, err: Allocator_Error) #no_bounds_check #optional_allocator_error { +inject_at_elem_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_broadcast arg: E, loc := #caller_location) -> (ok: bool, err: Allocator_Error) #no_bounds_check #optional_allocator_error { when !ODIN_NO_BOUNDS_CHECK { ensure(index >= 0, "Index must be positive.", loc) } if array == nil { return } - n := max(len(array), index) + old_len := len(array) + n := max(old_len, index) m :: 1 new_len := n + m - resize_soa(array, new_len, loc) or_return + // The tail shift and the stored element cover every new slot, + // except a gap of [old_len, index) when injecting past the end, which is + // zeroed explicitly below. + non_zero_resize_soa(array, new_len, loc) or_return when size_of(E) != 0 { ti := type_info_base(type_info_of(typeid_of(T))) si := &ti.variant.(Type_Info_Struct) - field_count := len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E) + FIELD_COUNT :: len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E) - item_offset := 0 - - arg_copy := arg - arg_ptr := &arg_copy - - for i in 0.. old_len { // zero the gap left by injecting past the end + mem_zero(rawptr(data + uintptr(old_len * type.size)), (index - old_len) * type.size) + } src := data + uintptr(index * type.size) dst := data + uintptr((index + m) * type.size) mem_copy(rawptr(dst), rawptr(src), (n - index) * type.size) - - mem_copy(rawptr(src), rawptr(uintptr(arg_ptr) + uintptr(item_offset)), type.size) - - item_offset += type.size } + + // store the new element via the compiler's #soa element store lowering + array[index] = arg } ok = true @@ -568,7 +811,7 @@ inject_at_elem_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_ // `inject_at_elems_soa` injects multiple elements in a dynamic SOA array at a specified index and moves the previous elements after that index "across" @builtin -inject_at_elems_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_broadcast args: ..E, loc := #caller_location) -> (ok: bool, err: Allocator_Error) #no_bounds_check #optional_allocator_error { +inject_at_elems_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_broadcast args: ..E, loc := #caller_location) -> (ok: bool, err: Allocator_Error) #no_bounds_check #optional_allocator_error { when !ODIN_NO_BOUNDS_CHECK { ensure(index >= 0, "Index must be positive.", loc) } @@ -580,11 +823,15 @@ inject_at_elems_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no return } - n := max(len(array), index) + old_len := len(array) + n := max(old_len, index) m := len(args) new_len := n + m - resize_soa(array, new_len, loc) or_return + // The tail shift and the stored elements cover every new slot, + // except a gap of [old_len, index) when injecting past the end, which is + // zeroed explicitly below. + non_zero_resize_soa(array, new_len, loc) or_return when size_of(E) != 0 { ti := type_info_base(type_info_of(typeid_of(T))) @@ -592,14 +839,28 @@ inject_at_elems_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no field_count := len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E) - item_offset := 0 - args_ptr := &args[0] + when !intrinsics.type_is_array(E) { + // E's field offsets come from E's own RTTI (si describes the SOA + // struct, whose fields are multipointers); basing on default struct + // layout here would misplace the fields of #packed elements. + se := &type_info_base(si.soa_base_type).variant.(Type_Info_Struct) + } + for i in 0.. old_len { // zero the gap left by injecting past the end + mem_zero(rawptr(data + uintptr(old_len * type.size)), (index - old_len) * type.size) + } src := data + uintptr(index * type.size) dst := data + uintptr((index + m) * type.size) @@ -607,11 +868,9 @@ inject_at_elems_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no for j in 0.. #soa[dynamic]E { // Note: If you the elements to remain in their order, use `ordered_remove_soa`. // Note: If the index is out of bounds, this procedure will panic. @builtin -unordered_remove_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, loc := #caller_location) #no_bounds_check { +unordered_remove_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int index: int, loc := #caller_location) #no_bounds_check { bounds_check_error_loc(loc, index, len(array)) if index+1 < len(array) { - ti := type_info_of(typeid_of(T)) - ti = type_info_base(ti) - si := &ti.variant.(Type_Info_Struct) - - field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)) - - data := uintptr(array) - for i in 0.. E) { + expect_same :: proc(t: ^testing.T, soa: #soa[dynamic]$T, model: [dynamic]T) { + testing.expect_value(t, len(soa), len(model)) + for i in 0.. 0 { + unordered_remove_soa(&soa, 0) + unordered_remove(&ref, 0) + } + testing.expect_value(t, len(soa), 0) + + // inject into an empty array + ok, err = inject_at_soa(&soa, 0, mk(998)) + testing.expect(t, ok) + testing.expect_value(t, err, nil) + inject_at(&ref, 0, mk(998)) + expect_same(t, soa, ref) + + // non-zero append + n, err = non_zero_append(&soa, mk(42)) + testing.expect_value(t, n, 1) + testing.expect_value(t, err, nil) + non_zero_append(&ref, mk(42)) + expect_same(t, soa, ref) + } + + // mixed field widths + padding + Padded :: struct { a: u8, b: u64, c: u16 } + check(t, Padded, proc(i: int) -> Padded { return {u8(i*3), u64(i)*257 + 7, u16(i*5 + 1)} }) + // array element type + check(t, [4]u16, proc(i: int) -> [4]u16 { return {u16(i), u16(i + 1), u16(i*3), u16(i*7)} }) + // eight fields at varying widths, no two neighbouring fields share a stride + Eight :: struct { a: u8, b: u16, c: u32, d: u64, e: i8, f: i16, g: f32, h: f64 } + check(t, Eight, proc(i: int) -> Eight { + return { + u8(i), u16(i*3 + 1), u32(i)*5 + 2, u64(i)*7 + 3, + i8(i >> 1), i16(i*11 + 4), f32(i)*1.5, f64(i)*2.25, + } + }) + // #packed struct with > 16 fields, field offsets diverging from default + // layout (u8/u64 alternate, align_of == 1), and the field count takes + // the type-erased batch appends compile time branch. + Packed17 :: struct #packed { + f0: u8, f1: u64, f2: u8, f3: u64, f4: u8, f5: u64, f6: u8, f7: u64, + f8: u8, f9: u64, f10: u8, f11: u64, f12: u8, f13: u64, f14: u8, f15: u64, + f16: u8, + } + check(t, Packed17, proc(i: int) -> Packed17 { + return { + u8(i), u64(i)*3 + 1, u8(i >> 1), u64(i)*5 + 2, u8(i >> 2), u64(i)*7 + 3, u8(i >> 3), u64(i)*11 + 4, + u8(i >> 4), u64(i)*13 + 5, u8(i >> 5), u64(i)*17 + 6, u8(i >> 6), u64(i)*19 + 7, u8(i >> 7), u64(i)*23 + 8, + u8(i*3), + } + }) +} diff --git a/tests/core/speed.odin b/tests/core/speed.odin index 0da7c1b82..c043aefe7 100644 --- a/tests/core/speed.odin +++ b/tests/core/speed.odin @@ -5,4 +5,5 @@ package tests_core @(require) import "crypto/bigint" @(require) import "hash" @(require) import "image" -@(require) import "math/big" \ No newline at end of file +@(require) import "math/big" +@(require) import "runtime" \ No newline at end of file