diff --git a/core/hash/xxhash/common.odin b/core/hash/xxhash/common.odin index aac5af7e1..6733bb83e 100644 --- a/core/hash/xxhash/common.odin +++ b/core/hash/xxhash/common.odin @@ -47,22 +47,19 @@ Error :: enum { XXH_DISABLE_PREFETCH :: #config(XXH_DISABLE_PREFETCH, false) -/* - llvm.prefetch fails code generation on Linux. -*/ -when !XXH_DISABLE_PREFETCH && ODIN_OS == "windows" { - import "core:sys/llvm" - +when !XXH_DISABLE_PREFETCH { prefetch_address :: #force_inline proc(address: rawptr) { - llvm.prefetch(address, .Read, .High, .Data) + intrinsics.prefetch_read_data(address, /*high*/3) } - prefetch_offset :: #force_inline proc(address: rawptr, auto_cast offset: uintptr) { + prefetch_offset :: #force_inline proc(address: rawptr, #any_int offset: uintptr) { ptr := rawptr(uintptr(address) + offset) prefetch_address(ptr) } } else { - prefetch_address :: #force_inline proc(address: rawptr) {} - prefetch_offset :: #force_inline proc(address: rawptr, auto_cast offset: uintptr) {} + prefetch_address :: #force_inline proc(address: rawptr) { + } + prefetch_offset :: #force_inline proc(address: rawptr, #any_int offset: uintptr) { + } } prefetch :: proc { prefetch_address, prefetch_offset, } diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 51fda2458..67ac22c8b 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -46,6 +46,11 @@ fixed_point_div :: proc(lhs, rhs: $T, #const scale: uint) -> T where type_is fixed_point_mul_sat :: proc(lhs, rhs: $T, #const scale: uint) -> T where type_is_integer(T) --- fixed_point_div_sat :: proc(lhs, rhs: $T, #const scale: uint) -> T where type_is_integer(T) --- +prefetch_read_instruction :: proc(address: rawptr, #const locality: i32 /* 0..=3 */) --- +prefetch_read_data :: proc(address: rawptr, #const locality: i32 /* 0..=3 */) --- +prefetch_write_instruction :: proc(address: rawptr, #const locality: i32 /* 0..=3 */) --- +prefetch_write_data :: proc(address: rawptr, #const locality: i32 /* 0..=3 */) --- + // Compiler Hints expect :: proc(val, expected_val: T) -> T --- diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 1f3928bd8..a1fd8eb6c 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -2642,7 +2642,41 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 operand->type = x.type; } break; - + + case BuiltinProc_prefetch_read_instruction: + case BuiltinProc_prefetch_read_data: + case BuiltinProc_prefetch_write_instruction: + case BuiltinProc_prefetch_write_data: + { + operand->mode = Addressing_NoValue; + operand->type = nullptr; + + Operand x = {}; + Operand y = {}; + check_expr(c, &x, ce->args[0]); + check_expr(c, &y, ce->args[1]); + if (x.mode == Addressing_Invalid) { + return false; + } + if (y.mode == Addressing_Invalid) { + return false; + } + check_assignment(c, &x, t_rawptr, builtin_name); + if (x.mode == Addressing_Invalid) { + return false; + } + if (y.mode != Addressing_Constant && is_type_integer(y.type)) { + error(y.expr, "Second argument to '%.*s' representing the locality must be an integer in the range 0..=3", LIT(builtin_name)); + return false; + } + i64 locality = exact_value_to_i64(y.value); + if (!(0 <= locality && locality <= 3)) { + error(y.expr, "Second argument to '%.*s' representing the locality must be an integer in the range 0..=3", LIT(builtin_name)); + return false; + } + + } + break; case BuiltinProc_syscall: { diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 466e679c3..4c0338849 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -69,6 +69,11 @@ enum BuiltinProcId { BuiltinProc_volatile_store, BuiltinProc_volatile_load, + + BuiltinProc_prefetch_read_instruction, + BuiltinProc_prefetch_read_data, + BuiltinProc_prefetch_write_instruction, + BuiltinProc_prefetch_write_data, BuiltinProc_atomic_fence, BuiltinProc_atomic_fence_acq, @@ -305,6 +310,11 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("volatile_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("volatile_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + + {STR_LIT("prefetch_read_instruction"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + {STR_LIT("prefetch_read_data"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + {STR_LIT("prefetch_write_instruction"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + {STR_LIT("prefetch_write_data"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("atomic_fence"), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("atomic_fence_acq"), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index ffbb532f0..4afaa4126 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1831,6 +1831,54 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, return lb_emit_conv(p, res, t); } + case BuiltinProc_prefetch_read_instruction: + case BuiltinProc_prefetch_read_data: + case BuiltinProc_prefetch_write_instruction: + case BuiltinProc_prefetch_write_data: + { + lbValue ptr = lb_emit_conv(p, lb_build_expr(p, ce->args[0]), t_rawptr); + unsigned long long locality = cast(unsigned long long)exact_value_to_i64(ce->args[1]->tav.value); + unsigned long long rw = 0; + unsigned long long cache = 0; + switch (id) { + case BuiltinProc_prefetch_read_instruction: + rw = 0; + cache = 0; + break; + case BuiltinProc_prefetch_read_data: + rw = 0; + cache = 1; + break; + case BuiltinProc_prefetch_write_instruction: + rw = 1; + cache = 0; + break; + case BuiltinProc_prefetch_write_data: + rw = 1; + cache = 1; + break; + } + + char const *name = "llvm.prefetch"; + + LLVMTypeRef types[1] = {lb_type(p->module, t_rawptr)}; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + LLVMTypeRef llvm_i32 = lb_type(p->module, t_i32); + LLVMValueRef args[4] = {}; + args[0] = ptr.value; + args[1] = LLVMConstInt(llvm_i32, rw, false); + args[2] = LLVMConstInt(llvm_i32, locality, false); + args[3] = LLVMConstInt(llvm_i32, cache, false); + + lbValue res = {}; + res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); + res.type = nullptr; + return res; + } + case BuiltinProc_syscall: { unsigned arg_count = cast(unsigned)ce->args.count;