Impl simd arm pmull

This commit is contained in:
Alexander Zhura
2026-08-01 16:22:15 +03:00
parent f62bafdc07
commit 659ba07cf3
2 changed files with 421 additions and 0 deletions

406
core/simd/arm/pmull.odin Normal file
View File

@@ -0,0 +1,406 @@
#+build arm64,arm32
package simd_arm
import "core:simd"
// Join two smaller vectors into a single larger vector
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_p8)
@(require_results, enable_target_feature = "neon")
vcombine_p8 :: #force_inline proc "c" (low, high: poly8x8_t) -> poly8x16_t {
when ODIN_ENDIAN == .Little {
return simd.shuffle(low, high, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)
} else {
low := simd.shuffle(low, low, 7, 6, 5, 4, 3, 2, 1, 0)
high := simd.shuffle(high, high, 7, 6, 5, 4, 3, 2, 1, 0)
c := simd.shuffle(low, high, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)
return simd.shuffle(c, c, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
}
}
// Join two smaller vectors into a single larger vector
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_p16)
@(require_results, enable_target_feature = "neon")
vcombine_p16 :: #force_inline proc "c" (low, high: poly16x4_t) -> poly16x8_t {
when ODIN_ENDIAN == .Little {
return simd.shuffle(low, high, 0, 1, 2, 3, 4, 5, 6, 7)
} else {
low := simd.shuffle(low, low, 3, 2, 1, 0)
high := simd.shuffle(high, high, 3, 2, 1, 0)
c := simd.shuffle(low, high, 0, 1, 2, 3, 4, 5, 6, 7)
return simd.shuffle(c, c, 7, 6, 5, 4, 3, 2, 1, 0)
}
}
// Join two smaller vectors into a single larger vector
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_p64)
@(require_results, enable_target_feature = "neon")
vcombine_p64 :: #force_inline proc "c" (low, high: poly64x1_t) -> poly64x2_t {
when ODIN_ENDIAN == .Little {
return simd.shuffle(low, high, 0, 1)
} else {
c := simd.shuffle(low, high, 0, 1)
return simd.shuffle(c, c, 1, 0)
}
}
// Insert vector element from another vector element
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p8)
@(require_results, enable_target_feature = "neon")
vset_lane_p8 :: #force_inline proc "c" (a: poly8_t, v: poly8x8_t, $LANE: int32_t) -> poly8x8_t where 0 <= LANE, LANE < 8 {
when ODIN_ENDIAN == .Little {
return simd.replace(v, LANE, a)
} else {
v := simd.shuffle(v, v, 7, 6, 5, 4, 3, 2, 1, 0)
c := simd.replace(v, LANE, a)
return simd.shuffle(c, c, 7, 6, 5, 4, 3, 2, 1, 0)
}
}
// Insert vector element from another vector element
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p16)
@(require_results, enable_target_feature = "neon")
vset_lane_p16 :: #force_inline proc "c" (a: poly16_t, v: poly16x4_t, $LANE: int32_t) -> poly16x4_t where 0 <= LANE, LANE < 4 {
when ODIN_ENDIAN == .Little {
return simd.replace(v, LANE, a)
} else {
v := simd.shuffle(v, v, 3, 2, 1, 0)
c := simd.replace(v, LANE, a)
return simd.shuffle(c, c, 3, 2, 1, 0)
}
}
// Insert vector element from another vector element
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p64)
@(require_results, enable_target_feature = "neon")
vset_lane_p64 :: #force_inline proc "c" (a: poly64_t, v: poly64x1_t, $LANE: int32_t) -> poly64x1_t where LANE == 0 {
return simd.replace(v, LANE, a)
}
// Move vector element to general-purpose register
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_p8)
@(require_results, enable_target_feature = "neon")
vget_lane_p8 :: #force_inline proc "c" (v: poly8x8_t, $LANE: int32_t) -> poly8_t where 0 <= LANE, LANE < 8 {
when ODIN_ENDIAN == .Little {
return simd.extract(v, LANE)
} else {
v := simd.shuffle(v, v, 7, 6, 5, 4, 3, 2, 1, 0)
return simd.extract(v, LANE)
}
}
// Move vector element to general-purpose register
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_p16)
@(require_results, enable_target_feature = "neon")
vget_lane_p16 :: #force_inline proc "c" (v: poly16x4_t, $LANE: int32_t) -> poly16_t where 0 <= LANE, LANE < 4 {
when ODIN_ENDIAN == .Little {
return simd.extract(v, LANE)
} else {
v := simd.shuffle(v, v, 3, 2, 1, 0)
return simd.extract(v, LANE)
}
}
// Move vector element to general-purpose register
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_p64)
@(require_results, enable_target_feature = "neon")
vget_lane_p64 :: #force_inline proc "c" (v: poly64x1_t, $LANE: int32_t) -> poly64_t where LANE == 0 {
return simd.extract(v, LANE)
}
// Insert vector element from another vector element
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p8)
@(require_results, enable_target_feature = "neon")
vsetq_lane_p8 :: #force_inline proc "c" (a: poly8_t, v: poly8x16_t, $LANE: int32_t) -> poly8x16_t where 0 <= LANE, LANE < 16 {
when ODIN_ENDIAN == .Little {
return simd.replace(v, LANE, a)
} else {
v := simd.shuffle(v, v, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
c := simd.replace(v, LANE, a)
return simd.shuffle(c, c, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
}
}
// Insert vector element from another vector element
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p16)
@(require_results, enable_target_feature = "neon")
vsetq_lane_p16 :: #force_inline proc "c" (a: poly16_t, v: poly16x8_t, $LANE: int32_t) -> poly16x8_t where 0 <= LANE, LANE < 8 {
when ODIN_ENDIAN == .Little {
return simd.replace(v, LANE, a)
} else {
v := simd.shuffle(v, v, 7, 6, 5, 4, 3, 2, 1, 0)
c := simd.replace(v, LANE, a)
return simd_shuffle(c, c, 7, 6, 5, 4, 3, 2, 1, 0)
}
}
// Insert vector element from another vector element
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p64)
@(require_results, enable_target_feature = "neon")
vsetq_lane_p64 :: #force_inline proc "c" (a: poly64_t, v: poly64x2_t, $LANE: int32_t) -> poly64x2_t where 0 <= LANE, LANE < 2 {
when ODIN_ENDIAN == .Little {
return simd.replace(v, LANE, a)
} else {
v := simd.shuffle(v, v, 1, 0)
c := simd.replace(v, LANE, a)
return simd.shuffle(c, c, 1, 0)
}
}
// Move vector element to general-purpose register
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_p8)
@(require_results, enable_target_feature = "neon")
vgetq_lane_p8 :: #force_inline proc "c" (v: poly8x16_t, $LANE: int32_t) -> poly8_t where 0 <= LANE, LANE < 16 {
when ODIN_ENDIAN == .Little {
return simd.extract(v, LANE)
} else {
v := simd.shuffle(v, v, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
return simd.extract(v, LANE)
}
}
// Move vector element to general-purpose register
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_p16)
@(require_results, enable_target_feature = "neon")
vgetq_lane_p16 :: #force_inline proc "c" (v: poly16x8_t, $LANE: int32_t) -> poly16_t where 0 <= LANE, LANE < 8 {
when ODIN_ENDIAN == .Little {
return simd.extract(v, LANE)
} else {
v := simd.shuffle(v, v, 7, 6, 5, 4, 3, 2, 1, 0)
return simd.extract(v, LANE)
}
}
// Move vector element to general-purpose register
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_p64)
@(require_results, enable_target_feature = "neon")
vgetq_lane_p64 :: #force_inline proc "c" (v: poly64x2_t, $LANE: int32_t) -> poly64_t where 0 <= LANE, LANE < 2 {
when ODIN_ENDIAN == .Little {
return simd.extract(v, LANE)
} else {
v := simd.shuffle(v, v, 1, 0)
return simd.extract(v, LANE)
}
}
// Duplicate vector element to vector or scalar
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_p8)
@(require_results, enable_target_feature = "neon")
vget_low_p8 :: #force_inline proc "c" (a: poly8x16_t) -> poly8x8_t {
when ODIN_ENDIAN == .Little {
return simd.shuffle(a, a, 0, 1, 2, 3, 4, 5, 6, 7)
} else {
a := simd.shuffle(a, a, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
b := simd.shuffle(a, a, 0, 1, 2, 3, 4, 5, 6, 7)
return simd.shuffle(b, b, 7, 6, 5, 4, 3, 2, 1, 0)
}
}
// Duplicate vector element to vector or scalar
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_p16)
@(require_results, enable_target_feature = "neon")
vget_low_p16 :: #force_inline proc "c" (a: poly16x8_t) -> poly16x4_t {
when ODIN_ENDIAN == .Little {
return simd.shuffle(a, a, 0, 1, 2, 3)
} else {
a := simd.shuffle(a, a, 7, 6, 5, 4, 3, 2, 1, 0)
b := simd.shuffle(a, a, 0, 1, 2, 3)
return simd.shuffle(b, b, 3, 2, 1, 0)
}
}
// Duplicate vector element to vector or scalar
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_p64)
@(require_results, enable_target_feature = "neon")
vget_low_p64 :: #force_inline proc "c" (a: poly64x2_t) -> poly64x1_t {
when ODIN_ENDIAN == .Little {
return transmute(poly64x1_t)simd.extract(a, 0)
} else {
a := simd.shuffle(a, a, 1, 0)
return transmute(poly64x1_t)simd.extract(a, 0)
}
}
// Duplicate vector element to vector or scalar
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_p8)
@(require_results, enable_target_feature = "neon")
vget_high_p8 :: #force_inline proc "c" (a: poly8x16_t) -> poly8x8_t {
when ODIN_ENDIAN == .Little {
return simd.shuffle(a, a, 8, 9, 10, 11, 12, 13, 14, 15)
} else {
a := simd.shuffle(a, a, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
b := simd.shuffle(a, a, 8, 9, 10, 11, 12, 13, 14, 15)
return simd.shuffle(b, b, 7, 6, 5, 4, 3, 2, 1, 0)
}
}
// Duplicate vector element to vector or scalar
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_p16)
@(require_results, enable_target_feature = "neon")
vget_high_p16 :: #force_inline proc "c" (a: poly16x8_t) -> poly16x4_t {
when ODIN_ENDIAN == .Little {
return simd.shuffle(a, a, 4, 5, 6, 7)
} else {
a := simd.shuffle(a, a, 7, 6, 5, 4, 3, 2, 1, 0)
b := simd.shuffle(a, a, 4, 5, 6, 7)
return simd.shuffle(b, b, 3, 2, 1, 0)
}
}
// Duplicate vector element to vector or scalar
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_p64)
@(require_results, enable_target_feature = "neon")
vget_high_p64 :: #force_inline proc "c" (a: poly64x2_t) -> poly64x1_t {
when ODIN_ENDIAN == .Little {
return transmute(poly64x1_t)simd.extract(a, 1)
} else {
a := simd.shuffle(a, a, 1, 0)
return transmute(poly64x1_t)simd.extract(a, 1)
}
}
// Bitwise exclusive OR
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p8)
@(require_results, enable_target_feature = "neon")
vadd_p8 :: #force_inline proc "c" (a, b: poly8x8_t) -> poly8x8_t {
return simd.bit_xor(a, b)
}
// Bitwise exclusive OR
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p16)
@(require_results, enable_target_feature = "neon")
vadd_p16 :: #force_inline proc "c" (a, b: poly16x4_t) -> poly16x4_t {
return simd.bit_xor(a, b)
}
// Bitwise exclusive OR
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p64)
@(require_results, enable_target_feature = "neon")
vadd_p64 :: #force_inline proc "c" (a, b: poly64x1_t) -> poly64x1_t {
return simd.bit_xor(a, b)
}
// Bitwise exclusive OR
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p8)
@(require_results, enable_target_feature = "neon")
vaddq_p8 :: #force_inline proc "c" (a, b: poly8x16_t) -> poly8x16_t {
return simd.bit_xor(a, b)
}
// Bitwise exclusive OR
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p16)
@(require_results, enable_target_feature = "neon")
vaddq_p16 :: #force_inline proc "c" (a, b: poly16x8_t) -> poly16x8_t {
return simd.bit_xor(a, b)
}
// Bitwise exclusive OR
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p64)
@(require_results, enable_target_feature = "neon")
vaddq_p64 :: #force_inline proc "c" (a, b: poly64x2_t) -> poly64x2_t {
return simd.bit_xor(a, b)
}
// Bitwise exclusive OR
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p128)
@(require_results, enable_target_feature = "neon")
vaddq_p128 :: #force_inline proc "c" (a, b: poly128_t) -> poly128_t {
return a ~ b
}
// Polynomial multiply
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_p8)
@(require_results, enable_target_feature = "neon")
vmul_p8 :: #force_inline proc "c" (a, b: poly8x8_t) -> poly8x8_t {
return _vmul_p8(a, b)
}
// Polynomial multiply
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_p8)
@(require_results, enable_target_feature = "neon")
vmulq_p8 :: #force_inline proc "c" (a, b: poly8x16_t) -> poly8x16_t {
return _vmulq_p8(a, b)
}
// Polynomial multiply long
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_p8)
@(require_results, enable_target_feature = "neon")
vmull_p8 :: #force_inline proc "c" (a, b: poly8x8_t) -> poly16x8_t {
return _vmull_p8(a, b)
}
when ODIN_ARCH == .arm64 {
// Polynomial multiply long
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_p8)
@(require_results, enable_target_feature = "neon")
vmull_high_p8 :: #force_inline proc "c" (a, b: poly8x16_t) -> poly16x8_t {
a := vget_high_p8(a)
b := vget_high_p8(b)
return vmull_p8(a, b)
}
// Polynomial multiply long
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_p64)
@(require_results, enable_target_feature = "neon,aes")
vmull_p64 :: #force_inline proc "c" (a, b: poly64_t) -> poly128_t {
return transmute(poly128_t)_vmull_p64(a, b)
}
// Polynomial multiply long
//
// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_p64)
@(require_results, enable_target_feature = "neon,aes")
vmull_high_p64 :: #force_inline proc "c" (a, b: poly64x2_t) -> poly128_t {
return vmull_p64(vgetq_lane_p64(a, 1), vgetq_lane_p64(b, 1))
}
}
@(private, default_calling_convention = "none")
foreign _ {
@(link_name = "llvm.aarch64.neon.pmul.v8i8" when ODIN_ARCH == .arm64 else "llvm.arm.neon.vmulp.v8i8")
_vmul_p8 :: proc(a, b: poly8x8_t) -> poly8x8_t ---
@(link_name = "llvm.aarch64.neon.pmul.v16i8" when ODIN_ARCH == .arm64 else "llvm.arm.neon.vmulp.v16i8")
_vmulq_p8 :: proc(a, b: poly8x16_t) -> poly8x16_t ---
@(link_name = "llvm.aarch64.neon.pmull.v8i16" when ODIN_ARCH == .arm64 else "llvm.arm.neon.vmullp.v8i16")
_vmull_p8 :: proc(a, b: poly8x8_t) -> poly16x8_t ---
}
when ODIN_ARCH == .arm64 {
@(private, default_calling_convention = "none")
foreign _ {
@(link_name = "llvm.aarch64.neon.pmull64")
_vmull_p64 :: proc(a, b: poly64_t) -> int8x16_t ---
}
}

View File

@@ -7,6 +7,21 @@ uint16_t :: u16
uint32_t :: u32
uint64_t :: u64
poly8_t :: u8
poly16_t :: u16
poly64_t :: u64
poly128_t :: u128
uint8x16_t :: #simd[16]u8
uint32x4_t :: #simd[4]u32
uint64x2_t :: #simd[2]u64
int32_t :: i32
int8x16_t :: #simd[16]i8
poly8x8_t :: #simd[8]poly8_t
poly8x16_t :: #simd[16]poly8_t
poly16x4_t :: #simd[4]poly16_t
poly16x8_t :: #simd[8]poly16_t
poly64x1_t :: #simd[1]poly64_t
poly64x2_t :: #simd[2]poly64_t