From 71932da7e37bfd13ddbbbbc96980e0274c44fbdf Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 23 Oct 2021 16:52:43 +0100 Subject: [PATCH] Add `intrinsics.unaligned_store` `intrinsics.unaligned_load` --- src/check_builtin.cpp | 4 + src/checker_builtin_procs.hpp | 6 ++ src/llvm_backend_proc.cpp | 138 ++++++++++++++++++++++------------ 3 files changed, 98 insertions(+), 50 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index a04302d01..a07c1f267 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -2567,6 +2567,8 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 case BuiltinProc_volatile_store: /*fallthrough*/ + case BuiltinProc_unaligned_store: + /*fallthrough*/ case BuiltinProc_atomic_store: case BuiltinProc_atomic_store_rel: case BuiltinProc_atomic_store_relaxed: @@ -2588,6 +2590,8 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 case BuiltinProc_volatile_load: /*fallthrough*/ + case BuiltinProc_unaligned_load: + /*fallthrough*/ case BuiltinProc_atomic_load: case BuiltinProc_atomic_load_acq: case BuiltinProc_atomic_load_relaxed: diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 8991d2d5c..e9eb76b0b 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -72,6 +72,9 @@ enum BuiltinProcId { BuiltinProc_volatile_store, BuiltinProc_volatile_load, + BuiltinProc_unaligned_store, + BuiltinProc_unaligned_load, + BuiltinProc_prefetch_read_instruction, BuiltinProc_prefetch_read_data, BuiltinProc_prefetch_write_instruction, @@ -314,6 +317,9 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("volatile_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("volatile_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("unaligned_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + {STR_LIT("unaligned_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("prefetch_read_instruction"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("prefetch_read_data"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("prefetch_write_instruction"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 222161164..e577dc248 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1,3 +1,62 @@ +void lb_mem_copy_overlapping(lbProcedure *p, lbValue dst, lbValue src, lbValue len, bool is_volatile=false) { + dst = lb_emit_conv(p, dst, t_rawptr); + src = lb_emit_conv(p, src, t_rawptr); + len = lb_emit_conv(p, len, t_int); + + char const *name = "llvm.memmove"; + if (LLVMIsConstant(len.value)) { + i64 const_len = cast(i64)LLVMConstIntGetSExtValue(len.value); + if (const_len <= 4*build_context.word_size) { + name = "llvm.memmove.inline"; + } + } + + LLVMTypeRef types[3] = { + lb_type(p->module, t_rawptr), + lb_type(p->module, t_rawptr), + lb_type(p->module, t_int) + }; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s.%s.%s", name, LLVMPrintTypeToString(types[0]), LLVMPrintTypeToString(types[1]), LLVMPrintTypeToString(types[2])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + LLVMValueRef args[4] = {}; + args[0] = dst.value; + args[1] = src.value; + args[2] = len.value; + args[3] = LLVMConstInt(LLVMInt1TypeInContext(p->module->ctx), 0, is_volatile); + LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); +} +void lb_mem_copy_non_overlapping(lbProcedure *p, lbValue dst, lbValue src, lbValue len, bool is_volatile=false) { + dst = lb_emit_conv(p, dst, t_rawptr); + src = lb_emit_conv(p, src, t_rawptr); + len = lb_emit_conv(p, len, t_int); + + char const *name = "llvm.memcpy"; + if (LLVMIsConstant(len.value)) { + i64 const_len = cast(i64)LLVMConstIntGetSExtValue(len.value); + if (const_len <= 4*build_context.word_size) { + name = "llvm.memcpy.inline"; + } + } + + LLVMTypeRef types[3] = { + lb_type(p->module, t_rawptr), + lb_type(p->module, t_rawptr), + lb_type(p->module, t_int) + }; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s.%s.%s", name, LLVMPrintTypeToString(types[0]), LLVMPrintTypeToString(types[1]), LLVMPrintTypeToString(types[2])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + LLVMValueRef args[4] = {}; + args[0] = dst.value; + args[1] = src.value; + args[2] = len.value; + args[3] = LLVMConstInt(LLVMInt1TypeInContext(p->module->ctx), 0, is_volatile); + LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); +} + lbProcedure *lb_create_procedure(lbModule *m, Entity *entity, bool ignore_body) { GB_ASSERT(entity != nullptr); @@ -1431,61 +1490,21 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, } case BuiltinProc_mem_copy: + { + lbValue dst = lb_build_expr(p, ce->args[0]); + lbValue src = lb_build_expr(p, ce->args[1]); + lbValue len = lb_build_expr(p, ce->args[2]); + + lb_mem_copy_overlapping(p, dst, src, len, false); + return {}; + } case BuiltinProc_mem_copy_non_overlapping: { lbValue dst = lb_build_expr(p, ce->args[0]); lbValue src = lb_build_expr(p, ce->args[1]); lbValue len = lb_build_expr(p, ce->args[2]); - dst = lb_emit_conv(p, dst, t_rawptr); - src = lb_emit_conv(p, src, t_rawptr); - len = lb_emit_conv(p, len, t_int); - - bool is_inlinable = false; - - if (ce->args[2]->tav.mode == Addressing_Constant) { - ExactValue ev = exact_value_to_integer(ce->args[2]->tav.value); - i64 const_len = exact_value_to_i64(ev); - // TODO(bill): Determine when it is better to do the `*.inline` versions - if (const_len <= 4*build_context.word_size) { - is_inlinable = true; - } - } - - char const *name = nullptr; - switch (id) { - case BuiltinProc_mem_copy: - if (is_inlinable) { - name = "llvm.memmove.inline"; - } else { - name = "llvm.memmove"; - } - break; - case BuiltinProc_mem_copy_non_overlapping: - if (is_inlinable) { - name = "llvm.memcpy.line"; - } else { - name = "llvm.memcpy"; - } - break; - } - - LLVMTypeRef types[3] = { - lb_type(p->module, t_rawptr), - lb_type(p->module, t_rawptr), - lb_type(p->module, t_int) - }; - unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); - GB_ASSERT_MSG(id != 0, "Unable to find %s.%s.%s.%s", name, LLVMPrintTypeToString(types[0]), LLVMPrintTypeToString(types[1]), LLVMPrintTypeToString(types[2])); - LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); - - LLVMValueRef args[4] = {}; - args[0] = dst.value; - args[1] = src.value; - args[2] = len.value; - args[3] = LLVMConstInt(LLVMInt1TypeInContext(p->module->ctx), 0, false); // is_volatile parameter - - LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); - + + lb_mem_copy_non_overlapping(p, dst, src, len, false); return {}; } @@ -1591,6 +1610,25 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, res.type = type_deref(dst.type); return res; } + + case BuiltinProc_unaligned_store: + { + lbValue dst = lb_build_expr(p, ce->args[0]); + lbValue src = lb_build_expr(p, ce->args[1]); + src = lb_address_from_load_or_generate_local(p, src); + Type *t = type_deref(dst.type); + lb_mem_copy_non_overlapping(p, dst, src, lb_const_int(p->module, t_int, type_size_of(t)), false); + return {}; + } + + case BuiltinProc_unaligned_load: + { + lbValue src = lb_build_expr(p, ce->args[0]); + Type *t = type_deref(src.type); + lbAddr dst = lb_add_local_generated(p, t, false); + lb_mem_copy_non_overlapping(p, dst.addr, src, lb_const_int(p->module, t_int, type_size_of(t)), false); + return lb_addr_load(p, dst); + } case BuiltinProc_atomic_add: case BuiltinProc_atomic_add_acq: