authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-03-22 20:22:15-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-03-22 20:22:15-04:00
logfe38d8142f713c1089cdc02f787184c48b8350b6
tree393cddfdb28389da0e50e8e879433eb151c0b7d2
parent62668e3e6b3c07d4145a7de3ebb9d5fdd728d75e

create multiple llvm.memcpy and llvm.memset with different align params

3 files changed, 67 insertions(+), 34 deletions(-)

src/all_types.hpp+9-2
......@@ -1415,6 +1415,8 @@ enum ZigLLVMFnId {
14151415 ZigLLVMFnIdOverflowArithmetic,
14161416 ZigLLVMFnIdFloor,
14171417 ZigLLVMFnIdCeil,
1418 ZigLLVMFnIdMemcpy,
1419 ZigLLVMFnIdMemset,
14181420};
14191421
14201422enum AddSubMul {
......@@ -1441,6 +1443,13 @@ struct ZigLLVMFnKey {
14411443 uint32_t bit_count;
14421444 bool is_signed;
14431445 } overflow_arithmetic;
1446 struct {
1447 uint32_t dest_align;
1448 uint32_t src_align;
1449 } memcpy;
1450 struct {
1451 uint32_t dest_align;
1452 } memset;
14441453 } data;
14451454};
14461455
......@@ -1629,8 +1638,6 @@ struct CodeGen {
16291638 ImportTableEntry *root_import;
16301639 ImportTableEntry *bootstrap_import;
16311640 ImportTableEntry *test_runner_import;
1632 LLVMValueRef memcpy_fn_val;
1633 LLVMValueRef memset_fn_val;
16341641 LLVMValueRef trap_fn_val;
16351642 LLVMValueRef return_address_fn_val;
16361643 LLVMValueRef frame_address_fn_val;
src/analyze.cpp+9
......@@ -5681,6 +5681,10 @@ uint32_t zig_llvm_fn_key_hash(ZigLLVMFnKey x) {
56815681 return ((uint32_t)(x.data.overflow_arithmetic.bit_count) * 87135777) +
56825682 ((uint32_t)(x.data.overflow_arithmetic.add_sub_mul) * 31640542) +
56835683 ((uint32_t)(x.data.overflow_arithmetic.is_signed) ? 1062315172 : 314955820);
5684 case ZigLLVMFnIdMemcpy:
5685 return x.data.memcpy.dest_align * 2325524557 + x.data.memcpy.src_align * 519976394;
5686 case ZigLLVMFnIdMemset:
5687 return x.data.memset.dest_align * 388171592;
56845688 }
56855689 zig_unreachable();
56865690}
......@@ -5700,6 +5704,11 @@ bool zig_llvm_fn_key_eql(ZigLLVMFnKey a, ZigLLVMFnKey b) {
57005704 return (a.data.overflow_arithmetic.bit_count == b.data.overflow_arithmetic.bit_count) &&
57015705 (a.data.overflow_arithmetic.add_sub_mul == b.data.overflow_arithmetic.add_sub_mul) &&
57025706 (a.data.overflow_arithmetic.is_signed == b.data.overflow_arithmetic.is_signed);
5707 case ZigLLVMFnIdMemcpy:
5708 return (a.data.memcpy.dest_align == b.data.memcpy.dest_align) &&
5709 (a.data.memcpy.src_align == b.data.memcpy.src_align);
5710 case ZigLLVMFnIdMemset:
5711 return (a.data.memset.dest_align == b.data.memset.dest_align);
57035712 }
57045713 zig_unreachable();
57055714}
src/codegen.cpp+49-32
......@@ -324,8 +324,12 @@ static void addLLVMFnAttrInt(LLVMValueRef fn_val, const char *attr_name, uint64_
324324 return addLLVMAttrInt(fn_val, -1, attr_name, attr_val);
325325}
326326
327static void addLLVMArgAttr(LLVMValueRef arg_val, unsigned param_index, const char *attr_name) {
328 return addLLVMAttr(arg_val, param_index + 1, attr_name);
327static void addLLVMArgAttr(LLVMValueRef fn_val, unsigned param_index, const char *attr_name) {
328 return addLLVMAttr(fn_val, param_index + 1, attr_name);
329}
330
331static void addLLVMArgAttrInt(LLVMValueRef fn_val, unsigned param_index, const char *attr_name, uint64_t attr_val) {
332 return addLLVMAttrInt(fn_val, param_index + 1, attr_name, attr_val);
329333}
330334
331335static void addLLVMCallsiteAttr(LLVMValueRef call_instr, unsigned param_index, const char *attr_name) {
......@@ -912,23 +916,31 @@ static void gen_safety_crash(CodeGen *g, PanicMsgId msg_id) {
912916 gen_panic(g, get_panic_msg_ptr_val(g, msg_id), nullptr);
913917}
914918
915static LLVMValueRef get_memcpy_fn_val(CodeGen *g) {
916 if (g->memcpy_fn_val)
917 return g->memcpy_fn_val;
919static LLVMValueRef get_memcpy_fn_val(CodeGen *g, uint32_t dest_align, uint32_t src_align) {
920 ZigLLVMFnKey key = {};
921 key.id = ZigLLVMFnIdMemcpy;
922 key.data.memcpy.dest_align = dest_align;
923 key.data.memcpy.src_align = src_align;
924
925 auto existing_entry = g->llvm_fn_table.maybe_get(key);
926 if (existing_entry)
927 return existing_entry->value;
918928
919929 LLVMTypeRef param_types[] = {
920930 LLVMPointerType(LLVMInt8Type(), 0),
921931 LLVMPointerType(LLVMInt8Type(), 0),
922932 LLVMIntType(g->pointer_size_bytes * 8),
923 LLVMInt32Type(),
924933 LLVMInt1Type(),
925934 };
926 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 5, false);
935 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 4, false);
927936 Buf *name = buf_sprintf("llvm.memcpy.p0i8.p0i8.i%d", g->pointer_size_bytes * 8);
928 g->memcpy_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
929 assert(LLVMGetIntrinsicID(g->memcpy_fn_val));
937 LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
938 addLLVMArgAttrInt(fn_val, 0, "align", dest_align);
939 addLLVMArgAttrInt(fn_val, 1, "align", src_align);
940 assert(LLVMGetIntrinsicID(fn_val));
930941
931 return g->memcpy_fn_val;
942 g->llvm_fn_table.put(key, fn_val);
943 return fn_val;
932944}
933945
934946static LLVMValueRef get_coro_destroy_fn_val(CodeGen *g) {
......@@ -1293,15 +1305,15 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {
12931305 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, err_name_val, slice_len_index, "");
12941306 LLVMValueRef err_name_len = gen_load_untyped(g, len_field_ptr, 0, false, "");
12951307
1308 LLVMValueRef memcpy_fn_val = get_memcpy_fn_val(g, u8_align_bytes, u8_align_bytes);
12961309 LLVMValueRef params[] = {
12971310 offset_buf_ptr, // dest pointer
12981311 err_name_ptr, // source pointer
12991312 err_name_len, // size bytes
1300 LLVMConstInt(LLVMInt32Type(), u8_align_bytes, false), // align bytes
13011313 LLVMConstNull(LLVMInt1Type()), // is volatile
13021314 };
13031315
1304 LLVMBuildCall(g->builder, get_memcpy_fn_val(g), params, 5, "");
1316 LLVMBuildCall(g->builder, memcpy_fn_val, params, 4, "");
13051317
13061318 LLVMValueRef const_prefix_len = LLVMConstInt(LLVMTypeOf(err_name_len), strlen(unwrap_err_msg_text), false);
13071319 LLVMValueRef full_buf_len = LLVMBuildNUWAdd(g->builder, const_prefix_len, err_name_len, "");
......@@ -1535,15 +1547,16 @@ static LLVMValueRef gen_assign_raw(CodeGen *g, LLVMValueRef ptr, TypeTableEntry
15351547 LLVMValueRef volatile_bit = ptr_type->data.pointer.is_volatile ?
15361548 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());
15371549
1550 LLVMValueRef memcpy_fn_val = get_memcpy_fn_val(g, align_bytes, align_bytes);
1551
15381552 LLVMValueRef params[] = {
15391553 dest_ptr, // dest pointer
15401554 src_ptr, // source pointer
15411555 LLVMConstInt(usize->type_ref, size_bytes, false),
1542 LLVMConstInt(LLVMInt32Type(), align_bytes, false),
15431556 volatile_bit,
15441557 };
15451558
1546 LLVMBuildCall(g->builder, get_memcpy_fn_val(g), params, 5, "");
1559 LLVMBuildCall(g->builder, memcpy_fn_val, params, 4, "");
15471560 return nullptr;
15481561 }
15491562
......@@ -2483,23 +2496,29 @@ static LLVMValueRef ir_render_bool_not(CodeGen *g, IrExecutable *executable, IrI
24832496 return LLVMBuildICmp(g->builder, LLVMIntEQ, value, zero, "");
24842497}
24852498
2486static LLVMValueRef get_memset_fn_val(CodeGen *g) {
2487 if (g->memset_fn_val)
2488 return g->memset_fn_val;
2499static LLVMValueRef get_memset_fn_val(CodeGen *g, uint32_t dest_align) {
2500 ZigLLVMFnKey key = {};
2501 key.id = ZigLLVMFnIdMemset;
2502 key.data.memset.dest_align = dest_align;
2503
2504 auto existing_entry = g->llvm_fn_table.maybe_get(key);
2505 if (existing_entry)
2506 return existing_entry->value;
24892507
24902508 LLVMTypeRef param_types[] = {
24912509 LLVMPointerType(LLVMInt8Type(), 0),
24922510 LLVMInt8Type(),
24932511 LLVMIntType(g->pointer_size_bytes * 8),
2494 LLVMInt32Type(),
24952512 LLVMInt1Type(),
24962513 };
2497 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 5, false);
2514 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 4, false);
24982515 Buf *name = buf_sprintf("llvm.memset.p0i8.i%d", g->pointer_size_bytes * 8);
2499 g->memset_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
2500 assert(LLVMGetIntrinsicID(g->memset_fn_val));
2516 LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
2517 addLLVMArgAttrInt(fn_val, 0, "align", dest_align);
2518 assert(LLVMGetIntrinsicID(fn_val));
25012519
2502 return g->memset_fn_val;
2520 g->llvm_fn_table.put(key, fn_val);
2521 return fn_val;
25032522}
25042523
25052524static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable,
......@@ -2535,21 +2554,21 @@ static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable,
25352554
25362555 assert(var->align_bytes > 0);
25372556
2557 LLVMValueRef memset_fn_val = get_memset_fn_val(g, var->align_bytes);
2558
25382559 // memset uninitialized memory to 0xa
25392560 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
25402561 LLVMValueRef fill_char = LLVMConstInt(LLVMInt8Type(), 0xaa, false);
25412562 LLVMValueRef dest_ptr = LLVMBuildBitCast(g->builder, var->value_ref, ptr_u8, "");
25422563 LLVMValueRef byte_count = LLVMConstInt(usize->type_ref, size_bytes, false);
2543 LLVMValueRef align_in_bytes = LLVMConstInt(LLVMInt32Type(), var->align_bytes, false);
25442564 LLVMValueRef params[] = {
25452565 dest_ptr,
25462566 fill_char,
25472567 byte_count,
2548 align_in_bytes,
25492568 LLVMConstNull(LLVMInt1Type()), // is volatile
25502569 };
25512570
2552 LLVMBuildCall(g->builder, get_memset_fn_val(g), params, 5, "");
2571 LLVMBuildCall(g->builder, memset_fn_val, params, 4, "");
25532572 }
25542573 }
25552574
......@@ -3399,17 +3418,16 @@ static LLVMValueRef ir_render_memset(CodeGen *g, IrExecutable *executable, IrIns
33993418 LLVMValueRef is_volatile = ptr_type->data.pointer.is_volatile ?
34003419 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());
34013420
3402 LLVMValueRef align_val = LLVMConstInt(LLVMInt32Type(), ptr_type->data.pointer.alignment, false);
3421 LLVMValueRef memset_fn_val = get_memset_fn_val(g, ptr_type->data.pointer.alignment);
34033422
34043423 LLVMValueRef params[] = {
34053424 dest_ptr_casted,
34063425 char_val,
34073426 len_val,
3408 align_val,
34093427 is_volatile,
34103428 };
34113429
3412 LLVMBuildCall(g->builder, get_memset_fn_val(g), params, 5, "");
3430 LLVMBuildCall(g->builder, memset_fn_val, params, 4, "");
34133431 return nullptr;
34143432}
34153433
......@@ -3432,18 +3450,17 @@ static LLVMValueRef ir_render_memcpy(CodeGen *g, IrExecutable *executable, IrIns
34323450 LLVMValueRef is_volatile = (dest_ptr_type->data.pointer.is_volatile || src_ptr_type->data.pointer.is_volatile) ?
34333451 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());
34343452
3435 uint32_t min_align_bytes = min(src_ptr_type->data.pointer.alignment, dest_ptr_type->data.pointer.alignment);
3436 LLVMValueRef align_val = LLVMConstInt(LLVMInt32Type(), min_align_bytes, false);
3453 LLVMValueRef memcpy_fn_val = get_memcpy_fn_val(g, dest_ptr_type->data.pointer.alignment,
3454 src_ptr_type->data.pointer.alignment);
34373455
34383456 LLVMValueRef params[] = {
34393457 dest_ptr_casted,
34403458 src_ptr_casted,
34413459 len_val,
3442 align_val,
34433460 is_volatile,
34443461 };
34453462
3446 LLVMBuildCall(g->builder, get_memcpy_fn_val(g), params, 5, "");
3463 LLVMBuildCall(g->builder, memcpy_fn_val, params, 4, "");
34473464 return nullptr;
34483465}
34493466