authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-10 15:33:11-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-10 15:46:42-05:00
log59375b3c22ff6694a20847e87a162705dbbc632a
treed3e35fb83c29a239cb9960a0e1e04fa7ac38f981
parent138a35df8f434115be04641b1df29514b0ef1cb8

llvm: workaround SROA misoptimizations in LLVM

Workaround #16392

1 files changed, 35 insertions(+), 25 deletions(-)

src/codegen/llvm.zig+35-25
......@@ -5137,7 +5137,7 @@ pub const FuncGen = struct {
51375137 try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
51385138
51395139 const alignment = return_type.abiAlignment(mod).toLlvm();
5140 const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);
5140 const ret_ptr = try self.buildAllocaWorkaround(return_type, alignment);
51415141 try llvm_args.append(ret_ptr);
51425142 break :blk ret_ptr;
51435143 };
......@@ -5186,7 +5186,7 @@ pub const FuncGen = struct {
51865186
51875187 const alignment = param_ty.abiAlignment(mod).toLlvm();
51885188 const param_llvm_ty = try o.lowerType(param_ty);
5189 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
5189 const arg_ptr = try self.buildAllocaWorkaround(param_ty, alignment);
51905190 if (isByRef(param_ty, mod)) {
51915191 const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
51925192 _ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
......@@ -5209,7 +5209,7 @@ pub const FuncGen = struct {
52095209 // LLVM does not allow bitcasting structs so we must allocate
52105210 // a local, store as one type, and then load as another type.
52115211 const alignment = param_ty.abiAlignment(mod).toLlvm();
5212 const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);
5212 const int_ptr = try self.buildAllocaWorkaround(param_ty, alignment);
52135213 _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
52145214 const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
52155215 try llvm_args.append(loaded);
......@@ -5552,7 +5552,7 @@ pub const FuncGen = struct {
55525552 const mod = o.module;
55535553
55545554 const result_alignment = va_list_ty.abiAlignment(mod).toLlvm();
5555 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
5555 const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);
55565556
55575557 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{}, &.{ dest_list, src_list }, "");
55585558 return if (isByRef(va_list_ty, mod))
......@@ -5576,7 +5576,7 @@ pub const FuncGen = struct {
55765576 const llvm_va_list_ty = try o.lowerType(va_list_ty);
55775577
55785578 const result_alignment = va_list_ty.abiAlignment(mod).toLlvm();
5579 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
5579 const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);
55805580
55815581 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{}, &.{dest_list}, "");
55825582 return if (isByRef(va_list_ty, mod))
......@@ -7407,7 +7407,7 @@ pub const FuncGen = struct {
74077407 self.ret_ptr
74087408 else brk: {
74097409 const alignment = optional_ty.abiAlignment(mod).toLlvm();
7410 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
7410 const optional_ptr = try self.buildAllocaWorkaround(optional_ty, alignment);
74117411 break :brk optional_ptr;
74127412 };
74137413
......@@ -7443,7 +7443,7 @@ pub const FuncGen = struct {
74437443 self.ret_ptr
74447444 else brk: {
74457445 const alignment = err_un_ty.abiAlignment(mod).toLlvm();
7446 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
7446 const result_ptr = try self.buildAllocaWorkaround(err_un_ty, alignment);
74477447 break :brk result_ptr;
74487448 };
74497449
......@@ -7481,7 +7481,7 @@ pub const FuncGen = struct {
74817481 self.ret_ptr
74827482 else brk: {
74837483 const alignment = err_un_ty.abiAlignment(mod).toLlvm();
7484 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
7484 const result_ptr = try self.buildAllocaWorkaround(err_un_ty, alignment);
74857485 break :brk result_ptr;
74867486 };
74877487
......@@ -7965,7 +7965,7 @@ pub const FuncGen = struct {
79657965
79667966 if (isByRef(inst_ty, mod)) {
79677967 const result_alignment = inst_ty.abiAlignment(mod).toLlvm();
7968 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
7968 const alloca_inst = try self.buildAllocaWorkaround(inst_ty, result_alignment);
79697969 {
79707970 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");
79717971 _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);
......@@ -8323,7 +8323,7 @@ pub const FuncGen = struct {
83238323
83248324 if (isByRef(dest_ty, mod)) {
83258325 const result_alignment = dest_ty.abiAlignment(mod).toLlvm();
8326 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
8326 const alloca_inst = try self.buildAllocaWorkaround(dest_ty, result_alignment);
83278327 {
83288328 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");
83298329 _ = try self.wip.store(.normal, result, field_ptr, result_alignment);
......@@ -8614,7 +8614,7 @@ pub const FuncGen = struct {
86148614 if (!result_is_ref) {
86158615 return self.dg.todo("implement bitcast vector to non-ref array", .{});
86168616 }
8617 const array_ptr = try self.buildAlloca(llvm_dest_ty, .default);
8617 const array_ptr = try self.buildAllocaWorkaround(inst_ty, .default);
86188618 const bitcast_ok = elem_ty.bitSize(mod) == elem_ty.abiSize(mod) * 8;
86198619 if (bitcast_ok) {
86208620 const alignment = inst_ty.abiAlignment(mod).toLlvm();
......@@ -8676,7 +8676,7 @@ pub const FuncGen = struct {
86768676
86778677 if (result_is_ref) {
86788678 const alignment = operand_ty.abiAlignment(mod).max(inst_ty.abiAlignment(mod)).toLlvm();
8679 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
8679 const result_ptr = try self.buildAllocaWorkaround(inst_ty, alignment);
86808680 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
86818681 return result_ptr;
86828682 }
......@@ -8688,7 +8688,7 @@ pub const FuncGen = struct {
86888688 // but LLVM won't let us bitcast struct values or vectors with padding bits.
86898689 // Therefore, we store operand to alloca, then load for result.
86908690 const alignment = operand_ty.abiAlignment(mod).max(inst_ty.abiAlignment(mod)).toLlvm();
8691 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
8691 const result_ptr = try self.buildAllocaWorkaround(inst_ty, alignment);
86928692 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
86938693 return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
86948694 }
......@@ -8753,9 +8753,9 @@ pub const FuncGen = struct {
87538753 if (!pointee_type.isFnOrHasRuntimeBitsIgnoreComptime(mod))
87548754 return (try o.lowerPtrToVoid(ptr_ty)).toValue();
87558755
8756 const pointee_llvm_ty = try o.lowerType(pointee_type);
8756 //const pointee_llvm_ty = try o.lowerType(pointee_type);
87578757 const alignment = ptr_ty.ptrAlignment(mod).toLlvm();
8758 return self.buildAlloca(pointee_llvm_ty, alignment);
8758 return self.buildAllocaWorkaround(pointee_type, alignment);
87598759 }
87608760
87618761 fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -8766,9 +8766,9 @@ pub const FuncGen = struct {
87668766 if (!ret_ty.isFnOrHasRuntimeBitsIgnoreComptime(mod))
87678767 return (try o.lowerPtrToVoid(ptr_ty)).toValue();
87688768 if (self.ret_ptr != .none) return self.ret_ptr;
8769 const ret_llvm_ty = try o.lowerType(ret_ty);
8769 //const ret_llvm_ty = try o.lowerType(ret_ty);
87708770 const alignment = ptr_ty.ptrAlignment(mod).toLlvm();
8771 return self.buildAlloca(ret_llvm_ty, alignment);
8771 return self.buildAllocaWorkaround(ret_ty, alignment);
87728772 }
87738773
87748774 /// Use this instead of builder.buildAlloca, because this function makes sure to
......@@ -8782,6 +8782,16 @@ pub const FuncGen = struct {
87828782 return buildAllocaInner(&self.wip, self.di_scope != null, llvm_ty, alignment, target);
87838783 }
87848784
8785 // Workaround for https://github.com/ziglang/zig/issues/16392
8786 fn buildAllocaWorkaround(
8787 self: *FuncGen,
8788 ty: Type,
8789 alignment: Builder.Alignment,
8790 ) Allocator.Error!Builder.Value {
8791 const o = self.dg.object;
8792 return self.buildAlloca(try o.builder.arrayType(ty.abiSize(o.module), .i8), alignment);
8793 }
8794
87858795 fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
87868796 const o = self.dg.object;
87878797 const mod = o.module;
......@@ -9736,7 +9746,7 @@ pub const FuncGen = struct {
97369746 const llvm_result_ty = accum_init.typeOfWip(&self.wip);
97379747
97389748 // Allocate and initialize our mutable variables
9739 const i_ptr = try self.buildAlloca(usize_ty, .default);
9749 const i_ptr = try self.buildAllocaWorkaround(Type.usize, .default);
97409750 _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
97419751 const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
97429752 _ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
......@@ -9948,7 +9958,7 @@ pub const FuncGen = struct {
99489958 // TODO in debug builds init to undef so that the padding will be 0xaa
99499959 // even if we fully populate the fields.
99509960 const alignment = result_ty.abiAlignment(mod).toLlvm();
9951 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
9961 const alloca_inst = try self.buildAllocaWorkaround(result_ty, alignment);
99529962
99539963 for (elements, 0..) |elem, i| {
99549964 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;
......@@ -9985,7 +9995,7 @@ pub const FuncGen = struct {
99859995 const llvm_usize = try o.lowerType(Type.usize);
99869996 const usize_zero = try o.builder.intValue(llvm_usize, 0);
99879997 const alignment = result_ty.abiAlignment(mod).toLlvm();
9988 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
9998 const alloca_inst = try self.buildAllocaWorkaround(result_ty, alignment);
99899999
999010000 const array_info = result_ty.arrayInfo(mod);
999110001 const elem_ptr_ty = try mod.ptrType(.{
......@@ -10061,7 +10071,7 @@ pub const FuncGen = struct {
1006110071 // We must construct the correct unnamed struct type here, in order to then set
1006210072 // the fields appropriately.
1006310073 const alignment = layout.abi_align.toLlvm();
10064 const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
10074 const result_ptr = try self.buildAllocaWorkaround(union_ty, alignment);
1006510075 const llvm_payload = try self.resolveInst(extra.init);
1006610076 const field_ty = union_obj.field_types.get(ip)[extra.field_index].toType();
1006710077 const field_llvm_ty = try o.lowerType(field_ty);
......@@ -10340,7 +10350,7 @@ pub const FuncGen = struct {
1034010350
1034110351 if (isByRef(optional_ty, mod)) {
1034210352 const payload_alignment = optional_ty.abiAlignment(mod).toLlvm();
10343 const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment);
10353 const alloca_inst = try self.buildAllocaWorkaround(optional_ty, payload_alignment);
1034410354
1034510355 {
1034610356 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");
......@@ -10481,9 +10491,9 @@ pub const FuncGen = struct {
1048110491 ) !Builder.Value {
1048210492 const o = fg.dg.object;
1048310493 const mod = o.module;
10484 const pointee_llvm_ty = try o.lowerType(pointee_type);
10494 //const pointee_llvm_ty = try o.lowerType(pointee_type);
1048510495 const result_align = InternPool.Alignment.fromLlvm(ptr_alignment).max(pointee_type.abiAlignment(mod)).toLlvm();
10486 const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
10496 const result_ptr = try fg.buildAllocaWorkaround(pointee_type, result_align);
1048710497 const size_bytes = pointee_type.abiSize(mod);
1048810498 _ = try fg.wip.callMemCpy(
1048910499 result_ptr,
......@@ -10542,7 +10552,7 @@ pub const FuncGen = struct {
1054210552
1054310553 if (isByRef(elem_ty, mod)) {
1054410554 const result_align = elem_ty.abiAlignment(mod).toLlvm();
10545 const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align);
10555 const result_ptr = try self.buildAllocaWorkaround(elem_ty, result_align);
1054610556
1054710557 const same_size_int = try o.builder.intType(@intCast(elem_bits));
1054810558 const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");