authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-03-22 20:22:15-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-03-22 20:22:15-04:00
logfe38d8142f713c1089cdc02f787184c48b8350b6
tree393cddfdb28389da0e50e8e879433eb151c0b7d2
parent62668e3e6b3c07d4145a7de3ebb9d5fdd728d75e

create multiple llvm.memcpy and llvm.memset with different align params

3 files changed, 67 insertions(+), 34 deletions(-)

src/all_types.hpp+9-2
...@@ -1415,6 +1415,8 @@ enum ZigLLVMFnId {...@@ -1415,6 +1415,8 @@ enum ZigLLVMFnId {
1415 ZigLLVMFnIdOverflowArithmetic,1415 ZigLLVMFnIdOverflowArithmetic,
1416 ZigLLVMFnIdFloor,1416 ZigLLVMFnIdFloor,
1417 ZigLLVMFnIdCeil,1417 ZigLLVMFnIdCeil,
1418 ZigLLVMFnIdMemcpy,
1419 ZigLLVMFnIdMemset,
1418};1420};
14191421
1420enum AddSubMul {1422enum AddSubMul {
...@@ -1441,6 +1443,13 @@ struct ZigLLVMFnKey {...@@ -1441,6 +1443,13 @@ struct ZigLLVMFnKey {
1441 uint32_t bit_count;1443 uint32_t bit_count;
1442 bool is_signed;1444 bool is_signed;
1443 } overflow_arithmetic;1445 } overflow_arithmetic;
1446 struct {
1447 uint32_t dest_align;
1448 uint32_t src_align;
1449 } memcpy;
1450 struct {
1451 uint32_t dest_align;
1452 } memset;
1444 } data;1453 } data;
1445};1454};
14461455
...@@ -1629,8 +1638,6 @@ struct CodeGen {...@@ -1629,8 +1638,6 @@ struct CodeGen {
1629 ImportTableEntry *root_import;1638 ImportTableEntry *root_import;
1630 ImportTableEntry *bootstrap_import;1639 ImportTableEntry *bootstrap_import;
1631 ImportTableEntry *test_runner_import;1640 ImportTableEntry *test_runner_import;
1632 LLVMValueRef memcpy_fn_val;
1633 LLVMValueRef memset_fn_val;
1634 LLVMValueRef trap_fn_val;1641 LLVMValueRef trap_fn_val;
1635 LLVMValueRef return_address_fn_val;1642 LLVMValueRef return_address_fn_val;
1636 LLVMValueRef frame_address_fn_val;1643 LLVMValueRef frame_address_fn_val;
src/analyze.cpp+9
...@@ -5681,6 +5681,10 @@ uint32_t zig_llvm_fn_key_hash(ZigLLVMFnKey x) {...@@ -5681,6 +5681,10 @@ uint32_t zig_llvm_fn_key_hash(ZigLLVMFnKey x) {
5681 return ((uint32_t)(x.data.overflow_arithmetic.bit_count) * 87135777) +5681 return ((uint32_t)(x.data.overflow_arithmetic.bit_count) * 87135777) +
5682 ((uint32_t)(x.data.overflow_arithmetic.add_sub_mul) * 31640542) +5682 ((uint32_t)(x.data.overflow_arithmetic.add_sub_mul) * 31640542) +
5683 ((uint32_t)(x.data.overflow_arithmetic.is_signed) ? 1062315172 : 314955820);5683 ((uint32_t)(x.data.overflow_arithmetic.is_signed) ? 1062315172 : 314955820);
5684 case ZigLLVMFnIdMemcpy:
5685 return x.data.memcpy.dest_align * 2325524557 + x.data.memcpy.src_align * 519976394;
5686 case ZigLLVMFnIdMemset:
5687 return x.data.memset.dest_align * 388171592;
5684 }5688 }
5685 zig_unreachable();5689 zig_unreachable();
5686}5690}
...@@ -5700,6 +5704,11 @@ bool zig_llvm_fn_key_eql(ZigLLVMFnKey a, ZigLLVMFnKey b) {...@@ -5700,6 +5704,11 @@ bool zig_llvm_fn_key_eql(ZigLLVMFnKey a, ZigLLVMFnKey b) {
5700 return (a.data.overflow_arithmetic.bit_count == b.data.overflow_arithmetic.bit_count) &&5704 return (a.data.overflow_arithmetic.bit_count == b.data.overflow_arithmetic.bit_count) &&
5701 (a.data.overflow_arithmetic.add_sub_mul == b.data.overflow_arithmetic.add_sub_mul) &&5705 (a.data.overflow_arithmetic.add_sub_mul == b.data.overflow_arithmetic.add_sub_mul) &&
5702 (a.data.overflow_arithmetic.is_signed == b.data.overflow_arithmetic.is_signed);5706 (a.data.overflow_arithmetic.is_signed == b.data.overflow_arithmetic.is_signed);
5707 case ZigLLVMFnIdMemcpy:
5708 return (a.data.memcpy.dest_align == b.data.memcpy.dest_align) &&
5709 (a.data.memcpy.src_align == b.data.memcpy.src_align);
5710 case ZigLLVMFnIdMemset:
5711 return (a.data.memset.dest_align == b.data.memset.dest_align);
5703 }5712 }
5704 zig_unreachable();5713 zig_unreachable();
5705}5714}
src/codegen.cpp+49-32
...@@ -324,8 +324,12 @@ static void addLLVMFnAttrInt(LLVMValueRef fn_val, const char *attr_name, uint64_...@@ -324,8 +324,12 @@ static void addLLVMFnAttrInt(LLVMValueRef fn_val, const char *attr_name, uint64_
324 return addLLVMAttrInt(fn_val, -1, attr_name, attr_val);324 return addLLVMAttrInt(fn_val, -1, attr_name, attr_val);
325}325}
326326
327static void addLLVMArgAttr(LLVMValueRef arg_val, unsigned param_index, const char *attr_name) {327static void addLLVMArgAttr(LLVMValueRef fn_val, unsigned param_index, const char *attr_name) {
328 return addLLVMAttr(arg_val, param_index + 1, attr_name);328 return addLLVMAttr(fn_val, param_index + 1, attr_name);
329}
330
331static void addLLVMArgAttrInt(LLVMValueRef fn_val, unsigned param_index, const char *attr_name, uint64_t attr_val) {
332 return addLLVMAttrInt(fn_val, param_index + 1, attr_name, attr_val);
329}333}
330334
331static void addLLVMCallsiteAttr(LLVMValueRef call_instr, unsigned param_index, const char *attr_name) {335static void addLLVMCallsiteAttr(LLVMValueRef call_instr, unsigned param_index, const char *attr_name) {
...@@ -912,23 +916,31 @@ static void gen_safety_crash(CodeGen *g, PanicMsgId msg_id) {...@@ -912,23 +916,31 @@ static void gen_safety_crash(CodeGen *g, PanicMsgId msg_id) {
912 gen_panic(g, get_panic_msg_ptr_val(g, msg_id), nullptr);916 gen_panic(g, get_panic_msg_ptr_val(g, msg_id), nullptr);
913}917}
914918
915static LLVMValueRef get_memcpy_fn_val(CodeGen *g) {919static LLVMValueRef get_memcpy_fn_val(CodeGen *g, uint32_t dest_align, uint32_t src_align) {
916 if (g->memcpy_fn_val)920 ZigLLVMFnKey key = {};
917 return g->memcpy_fn_val;921 key.id = ZigLLVMFnIdMemcpy;
922 key.data.memcpy.dest_align = dest_align;
923 key.data.memcpy.src_align = src_align;
924
925 auto existing_entry = g->llvm_fn_table.maybe_get(key);
926 if (existing_entry)
927 return existing_entry->value;
918928
919 LLVMTypeRef param_types[] = {929 LLVMTypeRef param_types[] = {
920 LLVMPointerType(LLVMInt8Type(), 0),930 LLVMPointerType(LLVMInt8Type(), 0),
921 LLVMPointerType(LLVMInt8Type(), 0),931 LLVMPointerType(LLVMInt8Type(), 0),
922 LLVMIntType(g->pointer_size_bytes * 8),932 LLVMIntType(g->pointer_size_bytes * 8),
923 LLVMInt32Type(),
924 LLVMInt1Type(),933 LLVMInt1Type(),
925 };934 };
926 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 5, false);935 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 4, false);
927 Buf *name = buf_sprintf("llvm.memcpy.p0i8.p0i8.i%d", g->pointer_size_bytes * 8);936 Buf *name = buf_sprintf("llvm.memcpy.p0i8.p0i8.i%d", g->pointer_size_bytes * 8);
928 g->memcpy_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);937 LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
929 assert(LLVMGetIntrinsicID(g->memcpy_fn_val));938 addLLVMArgAttrInt(fn_val, 0, "align", dest_align);
939 addLLVMArgAttrInt(fn_val, 1, "align", src_align);
940 assert(LLVMGetIntrinsicID(fn_val));
930941
931 return g->memcpy_fn_val;942 g->llvm_fn_table.put(key, fn_val);
943 return fn_val;
932}944}
933945
934static LLVMValueRef get_coro_destroy_fn_val(CodeGen *g) {946static LLVMValueRef get_coro_destroy_fn_val(CodeGen *g) {
...@@ -1293,15 +1305,15 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {...@@ -1293,15 +1305,15 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {
1293 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, err_name_val, slice_len_index, "");1305 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, err_name_val, slice_len_index, "");
1294 LLVMValueRef err_name_len = gen_load_untyped(g, len_field_ptr, 0, false, "");1306 LLVMValueRef err_name_len = gen_load_untyped(g, len_field_ptr, 0, false, "");
12951307
1308 LLVMValueRef memcpy_fn_val = get_memcpy_fn_val(g, u8_align_bytes, u8_align_bytes);
1296 LLVMValueRef params[] = {1309 LLVMValueRef params[] = {
1297 offset_buf_ptr, // dest pointer1310 offset_buf_ptr, // dest pointer
1298 err_name_ptr, // source pointer1311 err_name_ptr, // source pointer
1299 err_name_len, // size bytes1312 err_name_len, // size bytes
1300 LLVMConstInt(LLVMInt32Type(), u8_align_bytes, false), // align bytes
1301 LLVMConstNull(LLVMInt1Type()), // is volatile1313 LLVMConstNull(LLVMInt1Type()), // is volatile
1302 };1314 };
13031315
1304 LLVMBuildCall(g->builder, get_memcpy_fn_val(g), params, 5, "");1316 LLVMBuildCall(g->builder, memcpy_fn_val, params, 4, "");
13051317
1306 LLVMValueRef const_prefix_len = LLVMConstInt(LLVMTypeOf(err_name_len), strlen(unwrap_err_msg_text), false);1318 LLVMValueRef const_prefix_len = LLVMConstInt(LLVMTypeOf(err_name_len), strlen(unwrap_err_msg_text), false);
1307 LLVMValueRef full_buf_len = LLVMBuildNUWAdd(g->builder, const_prefix_len, err_name_len, "");1319 LLVMValueRef full_buf_len = LLVMBuildNUWAdd(g->builder, const_prefix_len, err_name_len, "");
...@@ -1535,15 +1547,16 @@ static LLVMValueRef gen_assign_raw(CodeGen *g, LLVMValueRef ptr, TypeTableEntry...@@ -1535,15 +1547,16 @@ static LLVMValueRef gen_assign_raw(CodeGen *g, LLVMValueRef ptr, TypeTableEntry
1535 LLVMValueRef volatile_bit = ptr_type->data.pointer.is_volatile ?1547 LLVMValueRef volatile_bit = ptr_type->data.pointer.is_volatile ?
1536 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());1548 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());
15371549
1550 LLVMValueRef memcpy_fn_val = get_memcpy_fn_val(g, align_bytes, align_bytes);
1551
1538 LLVMValueRef params[] = {1552 LLVMValueRef params[] = {
1539 dest_ptr, // dest pointer1553 dest_ptr, // dest pointer
1540 src_ptr, // source pointer1554 src_ptr, // source pointer
1541 LLVMConstInt(usize->type_ref, size_bytes, false),1555 LLVMConstInt(usize->type_ref, size_bytes, false),
1542 LLVMConstInt(LLVMInt32Type(), align_bytes, false),
1543 volatile_bit,1556 volatile_bit,
1544 };1557 };
15451558
1546 LLVMBuildCall(g->builder, get_memcpy_fn_val(g), params, 5, "");1559 LLVMBuildCall(g->builder, memcpy_fn_val, params, 4, "");
1547 return nullptr;1560 return nullptr;
1548 }1561 }
15491562
...@@ -2483,23 +2496,29 @@ static LLVMValueRef ir_render_bool_not(CodeGen *g, IrExecutable *executable, IrI...@@ -2483,23 +2496,29 @@ static LLVMValueRef ir_render_bool_not(CodeGen *g, IrExecutable *executable, IrI
2483 return LLVMBuildICmp(g->builder, LLVMIntEQ, value, zero, "");2496 return LLVMBuildICmp(g->builder, LLVMIntEQ, value, zero, "");
2484}2497}
24852498
2486static LLVMValueRef get_memset_fn_val(CodeGen *g) {2499static LLVMValueRef get_memset_fn_val(CodeGen *g, uint32_t dest_align) {
2487 if (g->memset_fn_val)2500 ZigLLVMFnKey key = {};
2488 return g->memset_fn_val;2501 key.id = ZigLLVMFnIdMemset;
2502 key.data.memset.dest_align = dest_align;
2503
2504 auto existing_entry = g->llvm_fn_table.maybe_get(key);
2505 if (existing_entry)
2506 return existing_entry->value;
24892507
2490 LLVMTypeRef param_types[] = {2508 LLVMTypeRef param_types[] = {
2491 LLVMPointerType(LLVMInt8Type(), 0),2509 LLVMPointerType(LLVMInt8Type(), 0),
2492 LLVMInt8Type(),2510 LLVMInt8Type(),
2493 LLVMIntType(g->pointer_size_bytes * 8),2511 LLVMIntType(g->pointer_size_bytes * 8),
2494 LLVMInt32Type(),
2495 LLVMInt1Type(),2512 LLVMInt1Type(),
2496 };2513 };
2497 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 5, false);2514 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 4, false);
2498 Buf *name = buf_sprintf("llvm.memset.p0i8.i%d", g->pointer_size_bytes * 8);2515 Buf *name = buf_sprintf("llvm.memset.p0i8.i%d", g->pointer_size_bytes * 8);
2499 g->memset_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);2516 LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
2500 assert(LLVMGetIntrinsicID(g->memset_fn_val));2517 addLLVMArgAttrInt(fn_val, 0, "align", dest_align);
2518 assert(LLVMGetIntrinsicID(fn_val));
25012519
2502 return g->memset_fn_val;2520 g->llvm_fn_table.put(key, fn_val);
2521 return fn_val;
2503}2522}
25042523
2505static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable,2524static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable,
...@@ -2535,21 +2554,21 @@ static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable,...@@ -2535,21 +2554,21 @@ static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable,
25352554
2536 assert(var->align_bytes > 0);2555 assert(var->align_bytes > 0);
25372556
2557 LLVMValueRef memset_fn_val = get_memset_fn_val(g, var->align_bytes);
2558
2538 // memset uninitialized memory to 0xa2559 // memset uninitialized memory to 0xa
2539 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);2560 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
2540 LLVMValueRef fill_char = LLVMConstInt(LLVMInt8Type(), 0xaa, false);2561 LLVMValueRef fill_char = LLVMConstInt(LLVMInt8Type(), 0xaa, false);
2541 LLVMValueRef dest_ptr = LLVMBuildBitCast(g->builder, var->value_ref, ptr_u8, "");2562 LLVMValueRef dest_ptr = LLVMBuildBitCast(g->builder, var->value_ref, ptr_u8, "");
2542 LLVMValueRef byte_count = LLVMConstInt(usize->type_ref, size_bytes, false);2563 LLVMValueRef byte_count = LLVMConstInt(usize->type_ref, size_bytes, false);
2543 LLVMValueRef align_in_bytes = LLVMConstInt(LLVMInt32Type(), var->align_bytes, false);
2544 LLVMValueRef params[] = {2564 LLVMValueRef params[] = {
2545 dest_ptr,2565 dest_ptr,
2546 fill_char,2566 fill_char,
2547 byte_count,2567 byte_count,
2548 align_in_bytes,
2549 LLVMConstNull(LLVMInt1Type()), // is volatile2568 LLVMConstNull(LLVMInt1Type()), // is volatile
2550 };2569 };
25512570
2552 LLVMBuildCall(g->builder, get_memset_fn_val(g), params, 5, "");2571 LLVMBuildCall(g->builder, memset_fn_val, params, 4, "");
2553 }2572 }
2554 }2573 }
25552574
...@@ -3399,17 +3418,16 @@ static LLVMValueRef ir_render_memset(CodeGen *g, IrExecutable *executable, IrIns...@@ -3399,17 +3418,16 @@ static LLVMValueRef ir_render_memset(CodeGen *g, IrExecutable *executable, IrIns
3399 LLVMValueRef is_volatile = ptr_type->data.pointer.is_volatile ?3418 LLVMValueRef is_volatile = ptr_type->data.pointer.is_volatile ?
3400 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());3419 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());
34013420
3402 LLVMValueRef align_val = LLVMConstInt(LLVMInt32Type(), ptr_type->data.pointer.alignment, false);3421 LLVMValueRef memset_fn_val = get_memset_fn_val(g, ptr_type->data.pointer.alignment);
34033422
3404 LLVMValueRef params[] = {3423 LLVMValueRef params[] = {
3405 dest_ptr_casted,3424 dest_ptr_casted,
3406 char_val,3425 char_val,
3407 len_val,3426 len_val,
3408 align_val,
3409 is_volatile,3427 is_volatile,
3410 };3428 };
34113429
3412 LLVMBuildCall(g->builder, get_memset_fn_val(g), params, 5, "");3430 LLVMBuildCall(g->builder, memset_fn_val, params, 4, "");
3413 return nullptr;3431 return nullptr;
3414}3432}
34153433
...@@ -3432,18 +3450,17 @@ static LLVMValueRef ir_render_memcpy(CodeGen *g, IrExecutable *executable, IrIns...@@ -3432,18 +3450,17 @@ static LLVMValueRef ir_render_memcpy(CodeGen *g, IrExecutable *executable, IrIns
3432 LLVMValueRef is_volatile = (dest_ptr_type->data.pointer.is_volatile || src_ptr_type->data.pointer.is_volatile) ?3450 LLVMValueRef is_volatile = (dest_ptr_type->data.pointer.is_volatile || src_ptr_type->data.pointer.is_volatile) ?
3433 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());3451 LLVMConstAllOnes(LLVMInt1Type()) : LLVMConstNull(LLVMInt1Type());
34343452
3435 uint32_t min_align_bytes = min(src_ptr_type->data.pointer.alignment, dest_ptr_type->data.pointer.alignment);3453 LLVMValueRef memcpy_fn_val = get_memcpy_fn_val(g, dest_ptr_type->data.pointer.alignment,
3436 LLVMValueRef align_val = LLVMConstInt(LLVMInt32Type(), min_align_bytes, false);3454 src_ptr_type->data.pointer.alignment);
34373455
3438 LLVMValueRef params[] = {3456 LLVMValueRef params[] = {
3439 dest_ptr_casted,3457 dest_ptr_casted,
3440 src_ptr_casted,3458 src_ptr_casted,
3441 len_val,3459 len_val,
3442 align_val,
3443 is_volatile,3460 is_volatile,
3444 };3461 };
34453462
3446 LLVMBuildCall(g->builder, get_memcpy_fn_val(g), params, 5, "");3463 LLVMBuildCall(g->builder, memcpy_fn_val, params, 4, "");
3447 return nullptr;3464 return nullptr;
3448}3465}
34493466