mirror of
https://github.com/odin-lang/Odin.git
synced 2026-08-14 09:44:26 +00:00
make soa builtins go vroom
This commit is contained in:
corley
@@ -50,13 +50,18 @@ Raw_SOA_Footer_Dynamic_Array :: struct {
|
||||
allocator: Allocator,
|
||||
}
|
||||
|
||||
// Note: When casting array to access footer/fields
|
||||
// uintptr(array) lowers to LLVM ptrtoint and captures pointer provenance,
|
||||
// whcih defeats #no_alias on the array in any code following (including in callers).
|
||||
// Multipointer indexing lowers to GEP and doesn't capture, so prefer that throughout.
|
||||
|
||||
@(builtin, require_results)
|
||||
raw_soa_footer_slice :: proc(array: ^$T/#soa[]$E) -> (footer: ^Raw_SOA_Footer_Slice) {
|
||||
if array == nil {
|
||||
return nil
|
||||
}
|
||||
field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E))
|
||||
footer = (^Raw_SOA_Footer_Slice)(uintptr(array) + field_count*size_of(rawptr))
|
||||
field_count := len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)
|
||||
footer = (^Raw_SOA_Footer_Slice)(&([^]byte)(array)[field_count*size_of(rawptr)])
|
||||
return
|
||||
}
|
||||
@(builtin, require_results)
|
||||
@@ -64,8 +69,8 @@ raw_soa_footer_dynamic_array :: proc(array: ^$T/#soa[dynamic]$E) -> (footer: ^Ra
|
||||
if array == nil {
|
||||
return nil
|
||||
}
|
||||
field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E))
|
||||
footer = (^Raw_SOA_Footer_Dynamic_Array)(uintptr(array) + field_count*size_of(rawptr))
|
||||
field_count := len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)
|
||||
footer = (^Raw_SOA_Footer_Dynamic_Array)(&([^]byte)(array)[field_count*size_of(rawptr)])
|
||||
return
|
||||
}
|
||||
raw_soa_footer :: proc{
|
||||
@@ -118,15 +123,13 @@ make_soa_aligned :: proc($T: typeid/#soa[]$E, #any_int length, alignment: int, a
|
||||
}
|
||||
new_data := raw_data(new_bytes)
|
||||
|
||||
data := uintptr(&array)
|
||||
offset := 0
|
||||
for i in 0..<field_count {
|
||||
type := si.types[i].variant.(Type_Info_Multi_Pointer).elem
|
||||
|
||||
offset = align_forward_int(offset, max_align)
|
||||
|
||||
(^uintptr)(data)^ = uintptr(new_data) + uintptr(offset)
|
||||
data += size_of(rawptr)
|
||||
([^]rawptr)(&array)[i] = rawptr(uintptr(new_data) + uintptr(offset))
|
||||
offset += type.size * length
|
||||
}
|
||||
footer.len = length
|
||||
@@ -174,7 +177,7 @@ make_soa :: proc{
|
||||
|
||||
|
||||
@builtin
|
||||
resize_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #caller_location) -> Allocator_Error {
|
||||
resize_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #caller_location) -> Allocator_Error {
|
||||
if array == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -215,7 +218,7 @@ resize_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #cal
|
||||
}
|
||||
|
||||
@builtin
|
||||
non_zero_resize_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #caller_location) -> Allocator_Error {
|
||||
non_zero_resize_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int length: int, loc := #caller_location) -> Allocator_Error {
|
||||
if array == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -225,6 +228,15 @@ non_zero_resize_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int length: int, lo
|
||||
return nil
|
||||
}
|
||||
|
||||
// `reserve_soa` will try to reserve memory of a passed SOA dynamic array to the requested element count (setting the `cap`).
|
||||
//
|
||||
// For maximizing performance it is recommended to avoid power-of-2 capacities
|
||||
// when manually reserving memory for SOA arrays, more so for sizes above 512.
|
||||
// For element types with several fields, prefer padding such capacities by at
|
||||
// least the number of fields that fit in a CPU cache line.
|
||||
// Modern cache lines are typically 64 (Intel/AMD/Arm64) or 128 bytes (Apple Silicon),
|
||||
// so pad requested capacity with e.g. `128 / size_of(field)` (for the smallest field when field sizes differ),
|
||||
// that is, prefer capacity of 4128 instead of 4096 for 4-byte fields.
|
||||
@builtin
|
||||
reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int capacity: int, loc := #caller_location) -> Allocator_Error {
|
||||
return _reserve_soa(array, capacity, true, loc)
|
||||
@@ -235,7 +247,33 @@ non_zero_reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int capacity: int,
|
||||
return _reserve_soa(array, capacity, false, loc)
|
||||
}
|
||||
|
||||
_reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, capacity: int, zero_memory: bool, loc := #caller_location) -> Allocator_Error {
|
||||
|
||||
// Note: capacity and performance
|
||||
// Each field is stored in contiguous memory with stride between fields capacity * size_of(field) (+ alignment).
|
||||
// When the stride is a multiple of 4KB, e.g. a power-of-2 capacity >= 1024 with 4-byte fields,
|
||||
// every same-size field's cache line for a given index maps to the same L1 set, and
|
||||
// per-element operations degrade (e.g. append_soa, inject_at_soa).
|
||||
// It is worst past 8 same-size fields where the burst exceeds the common
|
||||
// L1 8-way associativity and the fields collide (measured up to ~7-8x degradation
|
||||
// on a type with 18 same-size fields; fields of different sizes advance through
|
||||
// the sets at different rates, so they don't collide persistently).
|
||||
// Common SoA iterative flows may also be affected, when the loop touches multiple fields
|
||||
// and other hot data aliases the same L1 set.
|
||||
// So, for types with several fields, prefer padding such capacities by at
|
||||
// least the number of fields that fit in a CPU cache line.
|
||||
// Modern cache lines are typically 64 (Intel/AMD/Arm64) or 128 bytes (Apple Silicon),
|
||||
// so pad with e.g. 128 / size_of(field) (for the smallest field when field sizes differ),
|
||||
// so prefer 4128 instead of 4096 for 4-byte fields. Setting cap to e.g. 4097 is NOT enough
|
||||
// (it shifts field memory by only 4 bytes (same cache line) and fields keep colliding).
|
||||
// Note: 4KB stride in the discussion above is the x86 L1 set period (32–48KB/8–12-way);
|
||||
// Apple Silicon (128KB/8-way) has a 16KB period, so cap 4096 × 4 = 16KB stride
|
||||
// is still pathological there and the recommended 128-byte pad fixes it.
|
||||
// Note: struct size also plays a role, but the user facing padding recommendation
|
||||
// on reserve_soa() will handle up to 256 fields (of 4 bytes) decently on 8-way caches.
|
||||
// Note: Ideally, this should be handled internally by allowing allocated capacity to exceed requested capacity.
|
||||
// This would break current runtime tests though, they explicitly test requested cap.
|
||||
// Note: Capacities produced by append growth all the way from empty (2*cap + 8) are not affected.
|
||||
_reserve_soa :: #force_no_inline proc(array: ^$T/#soa[dynamic]$E, capacity: int, zero_memory: bool, loc := #caller_location) -> Allocator_Error {
|
||||
if array == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -322,7 +360,7 @@ _reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, capacity: int, zero_memory: boo
|
||||
|
||||
mem_copy(new_data_elem, old_data_elem, old_size_elem)
|
||||
|
||||
(^rawptr)(uintptr(array) + i*size_of(rawptr))^ = new_data_elem
|
||||
([^]rawptr)(array)[i] = new_data_elem
|
||||
|
||||
if zero_memory {
|
||||
mem_zero(rawptr(uintptr(new_data_elem) + uintptr(old_size_elem)), new_size_elem - old_size_elem)
|
||||
@@ -361,7 +399,7 @@ _reserve_soa :: proc(array: ^$T/#soa[dynamic]$E, capacity: int, zero_memory: boo
|
||||
|
||||
mem_copy(new_data_elem, old_data_elem, type.size * old_cap)
|
||||
|
||||
(^rawptr)(uintptr(array) + i*size_of(rawptr))^ = new_data_elem
|
||||
([^]rawptr)(array)[i] = new_data_elem
|
||||
|
||||
old_offset += type.size * old_cap
|
||||
new_offset += type.size * capacity
|
||||
@@ -388,7 +426,7 @@ non_zero_append_soa_elem :: proc(array: ^$T/#soa[dynamic]$E, #no_broadcast arg:
|
||||
return _append_soa_elem(array, false, arg, loc)
|
||||
}
|
||||
|
||||
_append_soa_elem :: proc(array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broadcast arg: E, loc := #caller_location) -> (n: int, err: Allocator_Error) #optional_allocator_error {
|
||||
_append_soa_elem :: proc(#no_alias array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broadcast arg: E, loc := #caller_location) -> (n: int, err: Allocator_Error) #optional_allocator_error {
|
||||
if array == nil {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -399,35 +437,13 @@ _append_soa_elem :: proc(array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broa
|
||||
err = _reserve_soa(array, cap, zero_memory, loc) // do not 'or_return' here as it could be a partial success
|
||||
}
|
||||
|
||||
footer := raw_soa_footer(array)
|
||||
|
||||
if size_of(E) > 0 && cap(array)-len(array) > 0 {
|
||||
ti := type_info_of(T)
|
||||
ti = type_info_base(ti)
|
||||
si := &ti.variant.(Type_Info_Struct)
|
||||
field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E))
|
||||
|
||||
data := (^rawptr)(array)^
|
||||
|
||||
soa_offset := 0
|
||||
item_offset := 0
|
||||
|
||||
arg_copy := arg
|
||||
arg_ptr := &arg_copy
|
||||
|
||||
max_align :: align_of(E)
|
||||
for i in 0..<field_count {
|
||||
type := si.types[i].variant.(Type_Info_Multi_Pointer).elem
|
||||
|
||||
soa_offset = align_forward_int(soa_offset, max_align)
|
||||
item_offset = align_forward_int(item_offset, type.align)
|
||||
|
||||
dst := rawptr(uintptr(data) + uintptr(soa_offset) + uintptr(type.size * footer.len))
|
||||
src := rawptr(uintptr(arg_ptr) + uintptr(item_offset))
|
||||
mem_copy(dst, src, type.size)
|
||||
|
||||
soa_offset += type.size * cap(array)
|
||||
item_offset += type.size
|
||||
footer := raw_soa_footer(array)
|
||||
// Field stores are generated by the compiler's #soa
|
||||
// element store lowering, specialized for E.
|
||||
// Note that #no_bounds_check is not optional, we write at index == len.
|
||||
#no_bounds_check {
|
||||
array[footer.len] = arg
|
||||
}
|
||||
footer.len += 1
|
||||
return 1, err
|
||||
@@ -446,7 +462,7 @@ non_zero_append_soa_elems :: proc(array: ^$T/#soa[dynamic]$E, #no_broadcast args
|
||||
}
|
||||
|
||||
|
||||
_append_soa_elems :: proc(array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broadcast args: []E, loc := #caller_location) -> (n: int, err: Allocator_Error) #optional_allocator_error {
|
||||
_append_soa_elems :: proc(#no_alias array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_broadcast args: []E, loc := #caller_location) -> (n: int, err: Allocator_Error) #optional_allocator_error {
|
||||
if array == nil {
|
||||
return
|
||||
}
|
||||
@@ -464,41 +480,267 @@ _append_soa_elems :: proc(array: ^$T/#soa[dynamic]$E, zero_memory: bool, #no_bro
|
||||
|
||||
footer := raw_soa_footer(array)
|
||||
if size_of(E) > 0 && arg_len > 0 {
|
||||
ti := type_info_of(typeid_of(T))
|
||||
ti = type_info_base(ti)
|
||||
si := &ti.variant.(Type_Info_Struct)
|
||||
field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E))
|
||||
|
||||
data := (^rawptr)(array)^
|
||||
|
||||
soa_offset := 0
|
||||
item_offset := 0
|
||||
|
||||
args_ptr := &args[0]
|
||||
|
||||
max_align :: align_of(E)
|
||||
for i in 0..<field_count {
|
||||
type := si.types[i].variant.(Type_Info_Multi_Pointer).elem
|
||||
|
||||
soa_offset = align_forward_int(soa_offset, max_align)
|
||||
item_offset = align_forward_int(item_offset, type.align)
|
||||
|
||||
dst := uintptr(data) + uintptr(soa_offset) + uintptr(type.size * footer.len)
|
||||
src := uintptr(args_ptr) + uintptr(item_offset)
|
||||
for j in 0..<arg_len {
|
||||
d := rawptr(dst + uintptr(j*type.size))
|
||||
s := rawptr(src + uintptr(j*size_of(E)))
|
||||
mem_copy(d, s, type.size)
|
||||
// For the common case where E has no more than 16 fields:
|
||||
// At ODIN_OPTIMIZATION_MODE >= .Speed do per-field copy passes that bind
|
||||
// the SOA struct's multipointers and each incoming element's fields by
|
||||
// position via expand_values with compile-time known types.
|
||||
// At ODIN_OPTIMIZATION_MODE <= .Size the compiler's lowering is
|
||||
// used, because it is the most compact at these field counts.
|
||||
// When E has >16 fields:
|
||||
// Type erased per-field loop using RTTI, with one mem_copy per element per field (codegen size is field count invariant).
|
||||
// (Note that offset and the multipointers must be read after _reserve_soa, because any growth moves them.)
|
||||
offset := footer.len
|
||||
FIELD_COUNT :: len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)
|
||||
when FIELD_COUNT <= 16 {
|
||||
when ODIN_OPTIMIZATION_MODE <= .Size {
|
||||
// Use the compiler's #soa element store lowering.
|
||||
// Note that #no_bounds_check is not optional, we write at index == len.
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len {
|
||||
array[offset + j] = args[j]
|
||||
}
|
||||
}
|
||||
} else when FIELD_COUNT >= 1 { // nothing to do if field count is 0
|
||||
_append_soa_elems_per_field_expanded(array, offset, args[:arg_len])
|
||||
}
|
||||
} else { // FIELD_COUNT > 16
|
||||
ti := type_info_base(type_info_of(typeid_of(T)))
|
||||
si := &ti.variant.(Type_Info_Struct)
|
||||
// si describes the SOA struct (fields are multipointers), so E's
|
||||
// field offsets are not available in it; read them from E's own RTTI (si.soa_base_type).
|
||||
// Note that basing on default struct layout here instead
|
||||
// would misplace the fields of #packed elements.
|
||||
// (> 16 fields is struct-only: array element types are capped at length 4.)
|
||||
se := &type_info_base(si.soa_base_type).variant.(Type_Info_Struct)
|
||||
|
||||
soa_offset += type.size * cap(array)
|
||||
item_offset += type.size
|
||||
src_base := uintptr(raw_data(args))
|
||||
for i in 0..<uintptr(FIELD_COUNT) {
|
||||
type := si.types[i].variant.(Type_Info_Multi_Pointer).elem
|
||||
|
||||
dst := uintptr(([^]rawptr)(array)[i]) + uintptr(type.size*offset)
|
||||
src := src_base + se.offsets[i]
|
||||
for j in 0..<arg_len {
|
||||
mem_copy(rawptr(dst + uintptr(j*type.size)), rawptr(src + uintptr(j*size_of(E))), type.size)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
footer.len += arg_len
|
||||
return arg_len, err
|
||||
}
|
||||
|
||||
// 1..16-field expnaded copy arms for _append_soa_elems.
|
||||
// this only exists to keep _append_soa_elems more readable.
|
||||
// args MUST already be clamped to capacity by the caller.
|
||||
_append_soa_elems_per_field_expanded :: #force_inline proc(#no_alias array: ^$T/#soa[dynamic]$E, offset: int, #no_broadcast args: []E) {
|
||||
arg_len := len(args)
|
||||
FIELD_COUNT :: len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)
|
||||
when FIELD_COUNT == 1 {
|
||||
// here and below the ignored last 3 values comprise the footer
|
||||
p0, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v := expand_values(args[j]); p0[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 2 {
|
||||
p0, p1, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v := expand_values(args[j]); p1[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 3 {
|
||||
p0, p1, p2, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v := expand_values(args[j]); p2[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 4 {
|
||||
p0, p1, p2, p3, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v := expand_values(args[j]); p3[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 5 {
|
||||
p0, p1, p2, p3, p4, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v := expand_values(args[j]); p4[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 6 {
|
||||
p0, p1, p2, p3, p4, p5, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v := expand_values(args[j]); p5[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 7 {
|
||||
p0, p1, p2, p3, p4, p5, p6, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v := expand_values(args[j]); p6[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 8 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v := expand_values(args[j]); p7[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 9 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, p8, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v, _ := expand_values(args[j]); p7[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, v := expand_values(args[j]); p8[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 10 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _, _, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v, _, _ := expand_values(args[j]); p7[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, v, _ := expand_values(args[j]); p8[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, v := expand_values(args[j]); p9[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 11 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _, _, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _, _, _, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v, _, _, _ := expand_values(args[j]); p7[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, v, _, _ := expand_values(args[j]); p8[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, v, _ := expand_values(args[j]); p9[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, v := expand_values(args[j]); p10[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 12 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _, _, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _, _, _, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _, _, _, _, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v, _, _, _, _ := expand_values(args[j]); p7[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, v, _, _, _ := expand_values(args[j]); p8[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, v, _, _ := expand_values(args[j]); p9[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, v, _ := expand_values(args[j]); p10[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, v := expand_values(args[j]); p11[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 13 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _, _, _, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _, _, _, _, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _, _, _, _, _, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v, _, _, _, _, _ := expand_values(args[j]); p7[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, v, _, _, _, _ := expand_values(args[j]); p8[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, v, _, _, _ := expand_values(args[j]); p9[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, v, _, _ := expand_values(args[j]); p10[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, v, _ := expand_values(args[j]); p11[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, v := expand_values(args[j]); p12[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 14 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _, _, _, _, _, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _, _, _, _, _, _, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v, _, _, _, _, _, _ := expand_values(args[j]); p7[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, v, _, _, _, _, _ := expand_values(args[j]); p8[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, v, _, _, _, _ := expand_values(args[j]); p9[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, v, _, _, _ := expand_values(args[j]); p10[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, v, _, _ := expand_values(args[j]); p11[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, v, _ := expand_values(args[j]); p12[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, _, v := expand_values(args[j]); p13[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 15 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _, _, _, _, _, _, _, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v, _, _, _, _, _, _, _ := expand_values(args[j]); p7[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, v, _, _, _, _, _, _ := expand_values(args[j]); p8[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, v, _, _, _, _, _ := expand_values(args[j]); p9[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, v, _, _, _, _ := expand_values(args[j]); p10[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, v, _, _, _ := expand_values(args[j]); p11[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, v, _, _ := expand_values(args[j]); p12[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, _, v, _ := expand_values(args[j]); p13[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, _, _, v := expand_values(args[j]); p14[offset+j] = v }
|
||||
}
|
||||
} else when FIELD_COUNT == 16 {
|
||||
p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, _, _, _ := expand_values(array^)
|
||||
#no_bounds_check {
|
||||
for j in 0..<arg_len { v, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p0[offset+j] = v }
|
||||
for j in 0..<arg_len { _, v, _, _, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p1[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, v, _, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p2[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, v, _, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p3[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, v, _, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p4[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, v, _, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p5[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, v, _, _, _, _, _, _, _, _, _ := expand_values(args[j]); p6[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, v, _, _, _, _, _, _, _, _ := expand_values(args[j]); p7[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, v, _, _, _, _, _, _, _ := expand_values(args[j]); p8[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, v, _, _, _, _, _, _ := expand_values(args[j]); p9[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, v, _, _, _, _, _ := expand_values(args[j]); p10[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, v, _, _, _, _ := expand_values(args[j]); p11[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, v, _, _, _ := expand_values(args[j]); p12[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, _, v, _, _ := expand_values(args[j]); p13[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, _, _, v, _ := expand_values(args[j]); p14[offset+j] = v }
|
||||
for j in 0..<arg_len { _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, v := expand_values(args[j]); p15[offset+j] = v }
|
||||
}
|
||||
} else {
|
||||
#panic("_append_soa_elems_per_field_expanded instantiated with an unsupported field count")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// The append_soa built-in procedure appends elements to the end of an #soa dynamic array
|
||||
@builtin
|
||||
@@ -523,43 +765,44 @@ append_nothing_soa :: proc(array: ^$T/#soa[dynamic]$E, loc := #caller_location)
|
||||
|
||||
// `inject_at_elem_soa` injects an element in a dynamic SOA array at a specified index and moves the previous elements after that index "across"
|
||||
@builtin
|
||||
inject_at_elem_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_broadcast arg: E, loc := #caller_location) -> (ok: bool, err: Allocator_Error) #no_bounds_check #optional_allocator_error {
|
||||
inject_at_elem_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_broadcast arg: E, loc := #caller_location) -> (ok: bool, err: Allocator_Error) #no_bounds_check #optional_allocator_error {
|
||||
when !ODIN_NO_BOUNDS_CHECK {
|
||||
ensure(index >= 0, "Index must be positive.", loc)
|
||||
}
|
||||
if array == nil {
|
||||
return
|
||||
}
|
||||
n := max(len(array), index)
|
||||
old_len := len(array)
|
||||
n := max(old_len, index)
|
||||
m :: 1
|
||||
new_len := n + m
|
||||
|
||||
resize_soa(array, new_len, loc) or_return
|
||||
// The tail shift and the stored element cover every new slot,
|
||||
// except a gap of [old_len, index) when injecting past the end, which is
|
||||
// zeroed explicitly below.
|
||||
non_zero_resize_soa(array, new_len, loc) or_return
|
||||
|
||||
when size_of(E) != 0 {
|
||||
ti := type_info_base(type_info_of(typeid_of(T)))
|
||||
si := &ti.variant.(Type_Info_Struct)
|
||||
|
||||
field_count := len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)
|
||||
FIELD_COUNT :: len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)
|
||||
|
||||
item_offset := 0
|
||||
|
||||
arg_copy := arg
|
||||
arg_ptr := &arg_copy
|
||||
|
||||
for i in 0..<field_count {
|
||||
data := (^uintptr)(uintptr(array) + uintptr(si.offsets[i]))^
|
||||
for i in 0..<FIELD_COUNT {
|
||||
data := uintptr(([^]rawptr)(array)[i])
|
||||
type := si.types[i].variant.(Type_Info_Multi_Pointer).elem
|
||||
item_offset = align_forward_int(item_offset, type.align)
|
||||
|
||||
if index > old_len { // zero the gap left by injecting past the end
|
||||
mem_zero(rawptr(data + uintptr(old_len * type.size)), (index - old_len) * type.size)
|
||||
}
|
||||
|
||||
src := data + uintptr(index * type.size)
|
||||
dst := data + uintptr((index + m) * type.size)
|
||||
mem_copy(rawptr(dst), rawptr(src), (n - index) * type.size)
|
||||
|
||||
mem_copy(rawptr(src), rawptr(uintptr(arg_ptr) + uintptr(item_offset)), type.size)
|
||||
|
||||
item_offset += type.size
|
||||
}
|
||||
|
||||
// store the new element via the compiler's #soa element store lowering
|
||||
array[index] = arg
|
||||
}
|
||||
|
||||
ok = true
|
||||
@@ -568,7 +811,7 @@ inject_at_elem_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_
|
||||
|
||||
// `inject_at_elems_soa` injects multiple elements in a dynamic SOA array at a specified index and moves the previous elements after that index "across"
|
||||
@builtin
|
||||
inject_at_elems_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_broadcast args: ..E, loc := #caller_location) -> (ok: bool, err: Allocator_Error) #no_bounds_check #optional_allocator_error {
|
||||
inject_at_elems_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int index: int, #no_broadcast args: ..E, loc := #caller_location) -> (ok: bool, err: Allocator_Error) #no_bounds_check #optional_allocator_error {
|
||||
when !ODIN_NO_BOUNDS_CHECK {
|
||||
ensure(index >= 0, "Index must be positive.", loc)
|
||||
}
|
||||
@@ -580,11 +823,15 @@ inject_at_elems_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no
|
||||
return
|
||||
}
|
||||
|
||||
n := max(len(array), index)
|
||||
old_len := len(array)
|
||||
n := max(old_len, index)
|
||||
m := len(args)
|
||||
new_len := n + m
|
||||
|
||||
resize_soa(array, new_len, loc) or_return
|
||||
// The tail shift and the stored elements cover every new slot,
|
||||
// except a gap of [old_len, index) when injecting past the end, which is
|
||||
// zeroed explicitly below.
|
||||
non_zero_resize_soa(array, new_len, loc) or_return
|
||||
|
||||
when size_of(E) != 0 {
|
||||
ti := type_info_base(type_info_of(typeid_of(T)))
|
||||
@@ -592,14 +839,28 @@ inject_at_elems_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no
|
||||
|
||||
field_count := len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E)
|
||||
|
||||
item_offset := 0
|
||||
|
||||
args_ptr := &args[0]
|
||||
|
||||
when !intrinsics.type_is_array(E) {
|
||||
// E's field offsets come from E's own RTTI (si describes the SOA
|
||||
// struct, whose fields are multipointers); basing on default struct
|
||||
// layout here would misplace the fields of #packed elements.
|
||||
se := &type_info_base(si.soa_base_type).variant.(Type_Info_Struct)
|
||||
}
|
||||
|
||||
for i in 0..<field_count {
|
||||
data := (^uintptr)(uintptr(array) + uintptr(si.offsets[i]))^
|
||||
data := uintptr(([^]rawptr)(array)[i])
|
||||
type := si.types[i].variant.(Type_Info_Multi_Pointer).elem
|
||||
item_offset = align_forward_int(item_offset, type.align)
|
||||
when intrinsics.type_is_array(E) {
|
||||
// array lanes are uniform, so offsets are just i * stride
|
||||
item_offset := uintptr(i * type.size)
|
||||
} else {
|
||||
item_offset := se.offsets[i]
|
||||
}
|
||||
|
||||
if index > old_len { // zero the gap left by injecting past the end
|
||||
mem_zero(rawptr(data + uintptr(old_len * type.size)), (index - old_len) * type.size)
|
||||
}
|
||||
|
||||
src := data + uintptr(index * type.size)
|
||||
dst := data + uintptr((index + m) * type.size)
|
||||
@@ -607,11 +868,9 @@ inject_at_elems_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, #no
|
||||
|
||||
for j in 0..<len(args) {
|
||||
d := rawptr(src + uintptr(j*type.size))
|
||||
s := rawptr(uintptr(args_ptr) + uintptr(item_offset) + uintptr(j*size_of(E)))
|
||||
s := rawptr(uintptr(args_ptr) + item_offset + uintptr(j*size_of(E)))
|
||||
mem_copy(d, s, type.size)
|
||||
}
|
||||
|
||||
item_offset += type.size
|
||||
}
|
||||
}
|
||||
|
||||
@@ -694,24 +953,11 @@ into_dynamic_soa :: proc(array: $T/#soa[]$E) -> #soa[dynamic]E {
|
||||
// Note: If you the elements to remain in their order, use `ordered_remove_soa`.
|
||||
// Note: If the index is out of bounds, this procedure will panic.
|
||||
@builtin
|
||||
unordered_remove_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, loc := #caller_location) #no_bounds_check {
|
||||
unordered_remove_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int index: int, loc := #caller_location) #no_bounds_check {
|
||||
bounds_check_error_loc(loc, index, len(array))
|
||||
if index+1 < len(array) {
|
||||
ti := type_info_of(typeid_of(T))
|
||||
ti = type_info_base(ti)
|
||||
si := &ti.variant.(Type_Info_Struct)
|
||||
|
||||
field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E))
|
||||
|
||||
data := uintptr(array)
|
||||
for i in 0..<field_count {
|
||||
type := si.types[i].variant.(Type_Info_Multi_Pointer).elem
|
||||
|
||||
offset := rawptr((^uintptr)(data)^ + uintptr(index*type.size))
|
||||
final := rawptr((^uintptr)(data)^ + uintptr((len(array)-1)*type.size))
|
||||
mem_copy(offset, final, type.size)
|
||||
data += size_of(rawptr)
|
||||
}
|
||||
// Use the compiler's #soa element load and store lowering.
|
||||
array[index] = array[len(array)-1]
|
||||
}
|
||||
raw_soa_footer_dynamic_array(array).len -= 1
|
||||
}
|
||||
@@ -722,23 +968,21 @@ unordered_remove_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, lo
|
||||
// Note: If you the elements do not have to remain in their order, prefer `unordered_remove_soa`.
|
||||
// Note: If the index is out of bounds, this procedure will panic.
|
||||
@builtin
|
||||
ordered_remove_soa :: proc(array: ^$T/#soa[dynamic]$E, #any_int index: int, loc := #caller_location) #no_bounds_check {
|
||||
ordered_remove_soa :: proc(#no_alias array: ^$T/#soa[dynamic]$E, #any_int index: int, loc := #caller_location) #no_bounds_check {
|
||||
bounds_check_error_loc(loc, index, len(array))
|
||||
if index+1 < len(array) {
|
||||
ti := type_info_of(typeid_of(T))
|
||||
ti = type_info_base(ti)
|
||||
si := &ti.variant.(Type_Info_Struct)
|
||||
|
||||
l1 := len(array)-1
|
||||
field_count := uintptr(len(E) when intrinsics.type_is_array(E) else intrinsics.type_struct_field_count(E))
|
||||
|
||||
data := uintptr(array)
|
||||
for i in 0..<field_count {
|
||||
type := si.types[i].variant.(Type_Info_Multi_Pointer).elem
|
||||
|
||||
offset := (^uintptr)(data)^ + uintptr(index*type.size)
|
||||
length := type.size*(len(array) - index - 1)
|
||||
offset := uintptr(([^]rawptr)(array)[i]) + uintptr(index*type.size)
|
||||
length := type.size*(l1 - index)
|
||||
mem_copy(rawptr(offset), rawptr(offset + uintptr(type.size)), length)
|
||||
data += size_of(rawptr)
|
||||
}
|
||||
}
|
||||
raw_soa_footer_dynamic_array(array).len -= 1
|
||||
|
||||
@@ -425,3 +425,196 @@ test_memory_compare_zero :: proc(t: ^testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Runs identical append/inject/remove sequences for a #soa[dynamic] array
|
||||
// and an AoS [dynamic] reference, comparing all elements after each
|
||||
// stage. Covers appends, injections (interior, at the end, and past the
|
||||
// end where the gap must read as zero elements), unordered and ordered
|
||||
// removes.
|
||||
@(test)
|
||||
test_soa_array_append_inject_remove :: proc(t: ^testing.T) {
|
||||
check :: proc(t: ^testing.T, $E: typeid, mk: proc(i: int) -> E) {
|
||||
expect_same :: proc(t: ^testing.T, soa: #soa[dynamic]$T, model: [dynamic]T) {
|
||||
testing.expect_value(t, len(soa), len(model))
|
||||
for i in 0..<min(len(soa), len(model)) {
|
||||
testing.expect_value(t, soa[i], model[i])
|
||||
}
|
||||
}
|
||||
|
||||
soa: #soa[dynamic]E
|
||||
defer delete(soa)
|
||||
ref: [dynamic]E
|
||||
defer delete(ref)
|
||||
|
||||
// single appends
|
||||
for i in 0..<10 {
|
||||
n, err := append(&soa, mk(i))
|
||||
testing.expect_value(t, n, 1)
|
||||
testing.expect_value(t, err, nil)
|
||||
append(&ref, mk(i))
|
||||
}
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// batch appends
|
||||
buf: [32]E
|
||||
for i in 0..<32 { buf[i] = mk(123 + i) }
|
||||
BATCH_LEN :: 5
|
||||
n, err := append(&soa, ..buf[:BATCH_LEN])
|
||||
testing.expect_value(t, n, BATCH_LEN)
|
||||
testing.expect_value(t, err, nil)
|
||||
append(&ref, ..buf[:BATCH_LEN])
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
n, err = append(&soa, ..buf[:])
|
||||
testing.expect_value(t, n, 32)
|
||||
testing.expect_value(t, err, nil)
|
||||
append(&ref, ..buf[:])
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// single injections
|
||||
ok: bool
|
||||
ok, err = inject_at_soa(&soa, 0, mk(300))
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, 0, mk(300))
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
ok, err = inject_at_soa(&soa, len(soa)/2, mk(301))
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, len(ref)/2, mk(301))
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// inject at the end
|
||||
ok, err = inject_at_soa(&soa, len(soa), mk(302))
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, len(ref), mk(302))
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// batch injections
|
||||
ok, err = inject_at_soa(&soa, 3, ..buf[:BATCH_LEN])
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, 3, ..buf[:BATCH_LEN])
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
ok, err = inject_at_soa(&soa, len(soa), ..buf[:])
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, len(ref), ..buf[:])
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// injecting nothing is a no-op
|
||||
ok, err = inject_at_soa(&soa, 4, ..buf[:0])
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, 4, ..buf[:0])
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// single injection past the end: the gap [len, index) reads as zero
|
||||
// elements. Poison the spare capacity first (fresh heap pages are
|
||||
// already zero, which would hide a missing gap zero), then shrink back.
|
||||
for i in 0..<8 {
|
||||
n, err = append(&soa, mk(900 + i))
|
||||
testing.expect_value(t, n, 1)
|
||||
testing.expect_value(t, err, nil)
|
||||
}
|
||||
testing.expect_value(t, resize_soa(&soa, len(soa) - 8), nil)
|
||||
ok, err = inject_at_soa(&soa, len(soa) + 3, mk(500))
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, len(ref) + 3, mk(500))
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// batch injection past the end (its gap slots reuse the poison above)
|
||||
ok, err = inject_at_soa(&soa, len(soa) + 2, ..buf[:BATCH_LEN])
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, len(ref) + 2, ..buf[:BATCH_LEN])
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// unordered removes
|
||||
unordered_remove_soa(&soa, 20)
|
||||
unordered_remove(&ref, 20)
|
||||
unordered_remove_soa(&soa, 0)
|
||||
unordered_remove(&ref, 0)
|
||||
unordered_remove_soa(&soa, len(soa)-1)
|
||||
unordered_remove(&ref, len(ref)-1)
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// ordered removes
|
||||
ordered_remove_soa(&soa, 17)
|
||||
ordered_remove(&ref, 17)
|
||||
ordered_remove_soa(&soa, 0)
|
||||
ordered_remove(&ref, 0)
|
||||
ordered_remove_soa(&soa, len(soa)-1)
|
||||
ordered_remove(&ref, len(ref)-1)
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// interleaved removes, appends and injections
|
||||
for i in 0..<8 {
|
||||
unordered_remove_soa(&soa, i)
|
||||
unordered_remove(&ref, i)
|
||||
n, err = append(&soa, mk(200 + i))
|
||||
testing.expect_value(t, n, 1)
|
||||
testing.expect_value(t, err, nil)
|
||||
append(&ref, mk(200 + i))
|
||||
ok, err = inject_at_soa(&soa, i, mk(400 + i))
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, i, mk(400 + i))
|
||||
}
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// remove till empty, then reuse
|
||||
for len(soa) > 0 {
|
||||
unordered_remove_soa(&soa, 0)
|
||||
unordered_remove(&ref, 0)
|
||||
}
|
||||
testing.expect_value(t, len(soa), 0)
|
||||
|
||||
// inject into an empty array
|
||||
ok, err = inject_at_soa(&soa, 0, mk(998))
|
||||
testing.expect(t, ok)
|
||||
testing.expect_value(t, err, nil)
|
||||
inject_at(&ref, 0, mk(998))
|
||||
expect_same(t, soa, ref)
|
||||
|
||||
// non-zero append
|
||||
n, err = non_zero_append(&soa, mk(42))
|
||||
testing.expect_value(t, n, 1)
|
||||
testing.expect_value(t, err, nil)
|
||||
non_zero_append(&ref, mk(42))
|
||||
expect_same(t, soa, ref)
|
||||
}
|
||||
|
||||
// mixed field widths + padding
|
||||
Padded :: struct { a: u8, b: u64, c: u16 }
|
||||
check(t, Padded, proc(i: int) -> Padded { return {u8(i*3), u64(i)*257 + 7, u16(i*5 + 1)} })
|
||||
// array element type
|
||||
check(t, [4]u16, proc(i: int) -> [4]u16 { return {u16(i), u16(i + 1), u16(i*3), u16(i*7)} })
|
||||
// eight fields at varying widths, no two neighbouring fields share a stride
|
||||
Eight :: struct { a: u8, b: u16, c: u32, d: u64, e: i8, f: i16, g: f32, h: f64 }
|
||||
check(t, Eight, proc(i: int) -> Eight {
|
||||
return {
|
||||
u8(i), u16(i*3 + 1), u32(i)*5 + 2, u64(i)*7 + 3,
|
||||
i8(i >> 1), i16(i*11 + 4), f32(i)*1.5, f64(i)*2.25,
|
||||
}
|
||||
})
|
||||
// #packed struct with > 16 fields, field offsets diverging from default
|
||||
// layout (u8/u64 alternate, align_of == 1), and the field count takes
|
||||
// the type-erased batch appends compile time branch.
|
||||
Packed17 :: struct #packed {
|
||||
f0: u8, f1: u64, f2: u8, f3: u64, f4: u8, f5: u64, f6: u8, f7: u64,
|
||||
f8: u8, f9: u64, f10: u8, f11: u64, f12: u8, f13: u64, f14: u8, f15: u64,
|
||||
f16: u8,
|
||||
}
|
||||
check(t, Packed17, proc(i: int) -> Packed17 {
|
||||
return {
|
||||
u8(i), u64(i)*3 + 1, u8(i >> 1), u64(i)*5 + 2, u8(i >> 2), u64(i)*7 + 3, u8(i >> 3), u64(i)*11 + 4,
|
||||
u8(i >> 4), u64(i)*13 + 5, u8(i >> 5), u64(i)*17 + 6, u8(i >> 6), u64(i)*19 + 7, u8(i >> 7), u64(i)*23 + 8,
|
||||
u8(i*3),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ package tests_core
|
||||
@(require) import "crypto/bigint"
|
||||
@(require) import "hash"
|
||||
@(require) import "image"
|
||||
@(require) import "math/big"
|
||||
@(require) import "math/big"
|
||||
@(require) import "runtime"
|
||||
Reference in New Issue
Block a user