authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-11-10 11:58:34-07:00
committergravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-11-10 12:22:40-07:00
log8f3880074fb76871d9a4f35d1f72d0304ac5b404
treef9d2cce5de2ef4080766064f853a9f0793ab5696
parentff699722da1f2df3e521c92cebe71c50910594d3

stage2: Be more strict about eliding loads

This change makes any of the `*_val` instructions check whether it's safe to elide copies for by-ref types rather than performing this elision blindly. AIR instructions fixed: - .array_elem_val - .struct_field_val - .unwrap_errunion_payload - .try - .optional_payload These now all respect value semantics, as expected. P.S. Thanks to Andrew for the new way to approach this. Many of the lines here are from his recommended change, which comes with the significant advantage that loads are now as small as the intervening memory access allows. Co-authored by: Andrew Kelley <andrew@ziglang.org>

5 files changed, 177 insertions(+), 75 deletions(-)

src/codegen/llvm.zig+118-75
...@@ -4568,14 +4568,14 @@ pub const FuncGen = struct {...@@ -4568,14 +4568,14 @@ pub const FuncGen = struct {
4568 .ret_addr => try self.airRetAddr(inst),4568 .ret_addr => try self.airRetAddr(inst),
4569 .frame_addr => try self.airFrameAddress(inst),4569 .frame_addr => try self.airFrameAddress(inst),
4570 .cond_br => try self.airCondBr(inst),4570 .cond_br => try self.airCondBr(inst),
4571 .@"try" => try self.airTry(inst),4571 .@"try" => try self.airTry(body[i..]),
4572 .try_ptr => try self.airTryPtr(inst),4572 .try_ptr => try self.airTryPtr(inst),
4573 .intcast => try self.airIntCast(inst),4573 .intcast => try self.airIntCast(inst),
4574 .trunc => try self.airTrunc(inst),4574 .trunc => try self.airTrunc(inst),
4575 .fptrunc => try self.airFptrunc(inst),4575 .fptrunc => try self.airFptrunc(inst),
4576 .fpext => try self.airFpext(inst),4576 .fpext => try self.airFpext(inst),
4577 .ptrtoint => try self.airPtrToInt(inst),4577 .ptrtoint => try self.airPtrToInt(inst),
4578 .load => try self.airLoad(inst, body, i + 1),4578 .load => try self.airLoad(body[i..]),
4579 .loop => try self.airLoop(inst),4579 .loop => try self.airLoop(inst),
4580 .not => try self.airNot(inst),4580 .not => try self.airNot(inst),
4581 .ret => try self.airRet(inst),4581 .ret => try self.airRet(inst),
...@@ -4634,7 +4634,7 @@ pub const FuncGen = struct {...@@ -4634,7 +4634,7 @@ pub const FuncGen = struct {
4634 .atomic_store_seq_cst => try self.airAtomicStore(inst, .SequentiallyConsistent),4634 .atomic_store_seq_cst => try self.airAtomicStore(inst, .SequentiallyConsistent),
46354635
4636 .struct_field_ptr => try self.airStructFieldPtr(inst),4636 .struct_field_ptr => try self.airStructFieldPtr(inst),
4637 .struct_field_val => try self.airStructFieldVal(inst),4637 .struct_field_val => try self.airStructFieldVal(body[i..]),
46384638
4639 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),4639 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
4640 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),4640 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
...@@ -4643,18 +4643,18 @@ pub const FuncGen = struct {...@@ -4643,18 +4643,18 @@ pub const FuncGen = struct {
46434643
4644 .field_parent_ptr => try self.airFieldParentPtr(inst),4644 .field_parent_ptr => try self.airFieldParentPtr(inst),
46454645
4646 .array_elem_val => try self.airArrayElemVal(inst),4646 .array_elem_val => try self.airArrayElemVal(body[i..]),
4647 .slice_elem_val => try self.airSliceElemVal(inst),4647 .slice_elem_val => try self.airSliceElemVal(body[i..]),
4648 .slice_elem_ptr => try self.airSliceElemPtr(inst),4648 .slice_elem_ptr => try self.airSliceElemPtr(inst),
4649 .ptr_elem_val => try self.airPtrElemVal(inst),4649 .ptr_elem_val => try self.airPtrElemVal(body[i..]),
4650 .ptr_elem_ptr => try self.airPtrElemPtr(inst),4650 .ptr_elem_ptr => try self.airPtrElemPtr(inst),
46514651
4652 .optional_payload => try self.airOptionalPayload(inst),4652 .optional_payload => try self.airOptionalPayload(body[i..]),
4653 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),4653 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
4654 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),4654 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
46554655
4656 .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),4656 .unwrap_errunion_payload => try self.airErrUnionPayload(body[i..], false),
4657 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),4657 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(body[i..], true),
4658 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),4658 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),
4659 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),4659 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
4660 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),4660 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
...@@ -5159,8 +5159,8 @@ pub const FuncGen = struct {...@@ -5159,8 +5159,8 @@ pub const FuncGen = struct {
5159 _ = self.builder.buildBr(end_block);5159 _ = self.builder.buildBr(end_block);
51605160
5161 self.builder.positionBuilderAtEnd(both_pl_block);5161 self.builder.positionBuilderAtEnd(both_pl_block);
5162 const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty);5162 const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true);
5163 const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty);5163 const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true);
5164 const payload_cmp = try self.cmp(lhs_payload, rhs_payload, payload_ty, op);5164 const payload_cmp = try self.cmp(lhs_payload, rhs_payload, payload_ty, op);
5165 _ = self.builder.buildBr(end_block);5165 _ = self.builder.buildBr(end_block);
5166 const both_pl_block_end = self.builder.getInsertBlock();5166 const both_pl_block_end = self.builder.getInsertBlock();
...@@ -5305,14 +5305,16 @@ pub const FuncGen = struct {...@@ -5305,14 +5305,16 @@ pub const FuncGen = struct {
5305 return null;5305 return null;
5306 }5306 }
53075307
5308 fn airTry(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {5308 fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index) !?*llvm.Value {
5309 const inst = body_tail[0];
5309 const pl_op = self.air.instructions.items(.data)[inst].pl_op;5310 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
5310 const err_union = try self.resolveInst(pl_op.operand);5311 const err_union = try self.resolveInst(pl_op.operand);
5311 const extra = self.air.extraData(Air.Try, pl_op.payload);5312 const extra = self.air.extraData(Air.Try, pl_op.payload);
5312 const body = self.air.extra[extra.end..][0..extra.data.body_len];5313 const body = self.air.extra[extra.end..][0..extra.data.body_len];
5313 const err_union_ty = self.air.typeOf(pl_op.operand);5314 const err_union_ty = self.air.typeOf(pl_op.operand);
5314 const result_ty = self.air.typeOfIndex(inst);5315 const payload_ty = self.air.typeOfIndex(inst);
5315 return lowerTry(self, err_union, body, err_union_ty, false, result_ty);5316 const can_elide_load = if (isByRef(payload_ty)) self.canElideLoad(body_tail) else false;
5317 return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, payload_ty);
5316 }5318 }
53175319
5318 fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {5320 fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
...@@ -5321,8 +5323,8 @@ pub const FuncGen = struct {...@@ -5321,8 +5323,8 @@ pub const FuncGen = struct {
5321 const err_union_ptr = try self.resolveInst(extra.data.ptr);5323 const err_union_ptr = try self.resolveInst(extra.data.ptr);
5322 const body = self.air.extra[extra.end..][0..extra.data.body_len];5324 const body = self.air.extra[extra.end..][0..extra.data.body_len];
5323 const err_union_ty = self.air.typeOf(extra.data.ptr).childType();5325 const err_union_ty = self.air.typeOf(extra.data.ptr).childType();
5324 const result_ty = self.air.typeOfIndex(inst);5326 const payload_ty = self.air.typeOfIndex(inst);
5325 return lowerTry(self, err_union_ptr, body, err_union_ty, true, result_ty);5327 return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, payload_ty);
5326 }5328 }
53275329
5328 fn lowerTry(5330 fn lowerTry(
...@@ -5331,6 +5333,7 @@ pub const FuncGen = struct {...@@ -5331,6 +5333,7 @@ pub const FuncGen = struct {
5331 body: []const Air.Inst.Index,5333 body: []const Air.Inst.Index,
5332 err_union_ty: Type,5334 err_union_ty: Type,
5333 operand_is_ptr: bool,5335 operand_is_ptr: bool,
5336 can_elide_load: bool,
5334 result_ty: Type,5337 result_ty: Type,
5335 ) !?*llvm.Value {5338 ) !?*llvm.Value {
5336 const payload_ty = err_union_ty.errorUnionPayload();5339 const payload_ty = err_union_ty.errorUnionPayload();
...@@ -5379,12 +5382,15 @@ pub const FuncGen = struct {...@@ -5379,12 +5382,15 @@ pub const FuncGen = struct {
5379 return fg.builder.buildBitCast(err_union, res_ptr_ty, "");5382 return fg.builder.buildBitCast(err_union, res_ptr_ty, "");
5380 }5383 }
5381 const offset = errUnionPayloadOffset(payload_ty, target);5384 const offset = errUnionPayloadOffset(payload_ty, target);
5382 if (operand_is_ptr or isByRef(payload_ty)) {5385 if (operand_is_ptr) {
5383 return fg.builder.buildStructGEP(err_union_llvm_ty, err_union, offset, "");5386 return fg.builder.buildStructGEP(err_union_llvm_ty, err_union, offset, "");
5384 } else if (isByRef(err_union_ty)) {5387 } else if (isByRef(err_union_ty)) {
5385 const payload_ptr = fg.builder.buildStructGEP(err_union_llvm_ty, err_union, offset, "");5388 const payload_ptr = fg.builder.buildStructGEP(err_union_llvm_ty, err_union, offset, "");
5386 if (isByRef(payload_ty)) {5389 if (isByRef(payload_ty)) {
5387 return payload_ptr;5390 if (can_elide_load)
5391 return payload_ptr;
5392
5393 return fg.loadByRef(payload_ptr, payload_ty, payload_ty.abiAlignment(target), false);
5388 }5394 }
5389 const load_inst = fg.builder.buildLoad(payload_ptr.getGEPResultElementType(), payload_ptr, "");5395 const load_inst = fg.builder.buildLoad(payload_ptr.getGEPResultElementType(), payload_ptr, "");
5390 load_inst.setAlignment(payload_ty.abiAlignment(target));5396 load_inst.setAlignment(payload_ty.abiAlignment(target));
...@@ -5625,14 +5631,16 @@ pub const FuncGen = struct {...@@ -5625,14 +5631,16 @@ pub const FuncGen = struct {
5625 return self.builder.buildStructGEP(slice_llvm_ty, slice_ptr, index, "");5631 return self.builder.buildStructGEP(slice_llvm_ty, slice_ptr, index, "");
5626 }5632 }
56275633
5628 fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {5634 fn airSliceElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !?*llvm.Value {
5635 const inst = body_tail[0];
5629 const bin_op = self.air.instructions.items(.data)[inst].bin_op;5636 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
5630 const slice_ty = self.air.typeOf(bin_op.lhs);5637 const slice_ty = self.air.typeOf(bin_op.lhs);
5631 if (!slice_ty.isVolatilePtr() and self.liveness.isUnused(inst)) return null;5638 if (!slice_ty.isVolatilePtr() and self.liveness.isUnused(inst)) return null;
56325639
5633 const slice = try self.resolveInst(bin_op.lhs);5640 const slice = try self.resolveInst(bin_op.lhs);
5634 const index = try self.resolveInst(bin_op.rhs);5641 const index = try self.resolveInst(bin_op.rhs);
5635 const llvm_elem_ty = try self.dg.lowerPtrElemTy(slice_ty.childType());5642 const elem_ty = slice_ty.childType();
5643 const llvm_elem_ty = try self.dg.lowerPtrElemTy(elem_ty);
5636 const base_ptr = self.builder.buildExtractValue(slice, 0, "");5644 const base_ptr = self.builder.buildExtractValue(slice, 0, "");
5637 const indices: [1]*llvm.Value = .{index};5645 const indices: [1]*llvm.Value = .{index};
5638 const ptr = self.builder.buildInBoundsGEP(llvm_elem_ty, base_ptr, &indices, indices.len, "");5646 const ptr = self.builder.buildInBoundsGEP(llvm_elem_ty, base_ptr, &indices, indices.len, "");
...@@ -5653,7 +5661,8 @@ pub const FuncGen = struct {...@@ -5653,7 +5661,8 @@ pub const FuncGen = struct {
5653 return self.builder.buildInBoundsGEP(llvm_elem_ty, base_ptr, &indices, indices.len, "");5661 return self.builder.buildInBoundsGEP(llvm_elem_ty, base_ptr, &indices, indices.len, "");
5654 }5662 }
56555663
5656 fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {5664 fn airArrayElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !?*llvm.Value {
5665 const inst = body_tail[0];
5657 if (self.liveness.isUnused(inst)) return null;5666 if (self.liveness.isUnused(inst)) return null;
56585667
5659 const bin_op = self.air.instructions.items(.data)[inst].bin_op;5668 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
...@@ -5666,7 +5675,11 @@ pub const FuncGen = struct {...@@ -5666,7 +5675,11 @@ pub const FuncGen = struct {
5666 const elem_ptr = self.builder.buildInBoundsGEP(array_llvm_ty, array_llvm_val, &indices, indices.len, "");5675 const elem_ptr = self.builder.buildInBoundsGEP(array_llvm_ty, array_llvm_val, &indices, indices.len, "");
5667 const elem_ty = array_ty.childType();5676 const elem_ty = array_ty.childType();
5668 if (isByRef(elem_ty)) {5677 if (isByRef(elem_ty)) {
5669 return elem_ptr;5678 if (canElideLoad(self, body_tail))
5679 return elem_ptr;
5680
5681 const target = self.dg.module.getTarget();
5682 return self.loadByRef(elem_ptr, elem_ty, elem_ty.abiAlignment(target), false);
5670 } else {5683 } else {
5671 const elem_llvm_ty = try self.dg.lowerType(elem_ty);5684 const elem_llvm_ty = try self.dg.lowerType(elem_ty);
5672 return self.builder.buildLoad(elem_llvm_ty, elem_ptr, "");5685 return self.builder.buildLoad(elem_llvm_ty, elem_ptr, "");
...@@ -5677,12 +5690,14 @@ pub const FuncGen = struct {...@@ -5677,12 +5690,14 @@ pub const FuncGen = struct {
5677 return self.builder.buildExtractElement(array_llvm_val, rhs, "");5690 return self.builder.buildExtractElement(array_llvm_val, rhs, "");
5678 }5691 }
56795692
5680 fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {5693 fn airPtrElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !?*llvm.Value {
5694 const inst = body_tail[0];
5681 const bin_op = self.air.instructions.items(.data)[inst].bin_op;5695 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
5682 const ptr_ty = self.air.typeOf(bin_op.lhs);5696 const ptr_ty = self.air.typeOf(bin_op.lhs);
5683 if (!ptr_ty.isVolatilePtr() and self.liveness.isUnused(inst)) return null;5697 if (!ptr_ty.isVolatilePtr() and self.liveness.isUnused(inst)) return null;
56845698
5685 const llvm_elem_ty = try self.dg.lowerPtrElemTy(ptr_ty.childType());5699 const elem_ty = ptr_ty.childType();
5700 const llvm_elem_ty = try self.dg.lowerPtrElemTy(elem_ty);
5686 const base_ptr = try self.resolveInst(bin_op.lhs);5701 const base_ptr = try self.resolveInst(bin_op.lhs);
5687 const rhs = try self.resolveInst(bin_op.rhs);5702 const rhs = try self.resolveInst(bin_op.rhs);
5688 // TODO: when we go fully opaque pointers in LLVM 16 we can remove this branch5703 // TODO: when we go fully opaque pointers in LLVM 16 we can remove this branch
...@@ -5743,7 +5758,8 @@ pub const FuncGen = struct {...@@ -5743,7 +5758,8 @@ pub const FuncGen = struct {
5743 return self.fieldPtr(inst, struct_ptr, struct_ptr_ty, field_index);5758 return self.fieldPtr(inst, struct_ptr, struct_ptr_ty, field_index);
5744 }5759 }
57455760
5746 fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {5761 fn airStructFieldVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !?*llvm.Value {
5762 const inst = body_tail[0];
5747 if (self.liveness.isUnused(inst)) return null;5763 if (self.liveness.isUnused(inst)) return null;
57485764
5749 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;5765 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
...@@ -5826,7 +5842,10 @@ pub const FuncGen = struct {...@@ -5826,7 +5842,10 @@ pub const FuncGen = struct {
5826 const llvm_field_ty = try self.dg.lowerType(field_ty);5842 const llvm_field_ty = try self.dg.lowerType(field_ty);
5827 const field_ptr = self.builder.buildBitCast(union_field_ptr, llvm_field_ty.pointerType(0), "");5843 const field_ptr = self.builder.buildBitCast(union_field_ptr, llvm_field_ty.pointerType(0), "");
5828 if (isByRef(field_ty)) {5844 if (isByRef(field_ty)) {
5829 return field_ptr;5845 if (canElideLoad(self, body_tail))
5846 return field_ptr;
5847
5848 return self.loadByRef(field_ptr, field_ty, layout.payload_align, false);
5830 } else {5849 } else {
5831 return self.builder.buildLoad(llvm_field_ty, field_ptr, "");5850 return self.builder.buildLoad(llvm_field_ty, field_ptr, "");
5832 }5851 }
...@@ -6516,7 +6535,8 @@ pub const FuncGen = struct {...@@ -6516,7 +6535,8 @@ pub const FuncGen = struct {
6516 return self.builder.buildStructGEP(optional_llvm_ty, operand, 0, "");6535 return self.builder.buildStructGEP(optional_llvm_ty, operand, 0, "");
6517 }6536 }
65186537
6519 fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {6538 fn airOptionalPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !?*llvm.Value {
6539 const inst = body_tail[0];
6520 if (self.liveness.isUnused(inst)) return null;6540 if (self.liveness.isUnused(inst)) return null;
65216541
6522 const ty_op = self.air.instructions.items(.data)[inst].ty_op;6542 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
...@@ -6531,14 +6551,16 @@ pub const FuncGen = struct {...@@ -6531,14 +6551,16 @@ pub const FuncGen = struct {
6531 }6551 }
65326552
6533 const opt_llvm_ty = try self.dg.lowerType(optional_ty);6553 const opt_llvm_ty = try self.dg.lowerType(optional_ty);
6534 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty);6554 const can_elide_load = if (isByRef(payload_ty)) self.canElideLoad(body_tail) else false;
6555 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, can_elide_load);
6535 }6556 }
65366557
6537 fn airErrUnionPayload(6558 fn airErrUnionPayload(
6538 self: *FuncGen,6559 self: *FuncGen,
6539 inst: Air.Inst.Index,6560 body_tail: []const Air.Inst.Index,
6540 operand_is_ptr: bool,6561 operand_is_ptr: bool,
6541 ) !?*llvm.Value {6562 ) !?*llvm.Value {
6563 const inst = body_tail[0];
6542 if (self.liveness.isUnused(inst)) return null;6564 if (self.liveness.isUnused(inst)) return null;
65436565
6544 const ty_op = self.air.instructions.items(.data)[inst].ty_op;6566 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
...@@ -6558,12 +6580,15 @@ pub const FuncGen = struct {...@@ -6558,12 +6580,15 @@ pub const FuncGen = struct {
6558 }6580 }
6559 const offset = errUnionPayloadOffset(payload_ty, target);6581 const offset = errUnionPayloadOffset(payload_ty, target);
6560 const err_union_llvm_ty = try self.dg.lowerType(err_union_ty);6582 const err_union_llvm_ty = try self.dg.lowerType(err_union_ty);
6561 if (operand_is_ptr or isByRef(payload_ty)) {6583 if (operand_is_ptr) {
6562 return self.builder.buildStructGEP(err_union_llvm_ty, operand, offset, "");6584 return self.builder.buildStructGEP(err_union_llvm_ty, operand, offset, "");
6563 } else if (isByRef(err_union_ty)) {6585 } else if (isByRef(err_union_ty)) {
6564 const payload_ptr = self.builder.buildStructGEP(err_union_llvm_ty, operand, offset, "");6586 const payload_ptr = self.builder.buildStructGEP(err_union_llvm_ty, operand, offset, "");
6565 if (isByRef(payload_ty)) {6587 if (isByRef(payload_ty)) {
6566 return payload_ptr;6588 if (self.canElideLoad(body_tail))
6589 return payload_ptr;
6590
6591 return self.loadByRef(payload_ptr, payload_ty, payload_ty.abiAlignment(target), false);
6567 }6592 }
6568 const load_inst = self.builder.buildLoad(payload_ptr.getGEPResultElementType(), payload_ptr, "");6593 const load_inst = self.builder.buildLoad(payload_ptr.getGEPResultElementType(), payload_ptr, "");
6569 load_inst.setAlignment(payload_ty.abiAlignment(target));6594 load_inst.setAlignment(payload_ty.abiAlignment(target));
...@@ -8064,35 +8089,37 @@ pub const FuncGen = struct {...@@ -8064,35 +8089,37 @@ pub const FuncGen = struct {
8064 return null;8089 return null;
8065 }8090 }
80668091
8067 fn airLoad(8092 /// As an optimization, we want to avoid unnecessary copies of isByRef=true
8068 self: *FuncGen,8093 /// types. Here, we scan forward in the current block, looking to see if
8069 inst: Air.Inst.Index,8094 /// this load dies before any side effects occur. In such case, we can
8070 body: []const Air.Inst.Index,8095 /// safely return the operand without making a copy.
8071 body_i: usize,8096 ///
8072 ) !?*llvm.Value {8097 /// The first instruction of `body_tail` is the one whose copy we want to elide.
8073 const ty_op = self.air.instructions.items(.data)[inst].ty_op;8098 fn canElideLoad(fg: *FuncGen, body_tail: []const Air.Inst.Index) bool {
8074 const ptr_ty = self.air.typeOf(ty_op.operand);8099 for (body_tail[1..]) |body_inst| {
8100 switch (fg.liveness.categorizeOperand(fg.air, body_inst, body_tail[0])) {
8101 .none => continue,
8102 .write, .noret, .complex => return false,
8103 .tomb => return true,
8104 }
8105 } else unreachable;
8106 }
8107
8108 fn airLoad(fg: *FuncGen, body_tail: []const Air.Inst.Index) !?*llvm.Value {
8109 const inst = body_tail[0];
8110 const ty_op = fg.air.instructions.items(.data)[inst].ty_op;
8111 const ptr_ty = fg.air.typeOf(ty_op.operand);
8112 const ptr_info = ptr_ty.ptrInfo().data;
8113 const ptr = try fg.resolveInst(ty_op.operand);
8114
8075 elide: {8115 elide: {
8076 const ptr_info = ptr_ty.ptrInfo().data;
8077 if (ptr_info.@"volatile") break :elide;8116 if (ptr_info.@"volatile") break :elide;
8078 if (self.liveness.isUnused(inst)) return null;8117 if (fg.liveness.isUnused(inst)) return null;
8079 if (!isByRef(ptr_info.pointee_type)) break :elide;8118 if (!isByRef(ptr_info.pointee_type)) break :elide;
80808119 if (!canElideLoad(fg, body_tail)) break :elide;
8081 // It would be valid to fall back to the code below here that simply calls8120 return ptr;
8082 // load(). However, as an optimization, we want to avoid unnecessary copies
8083 // of isByRef=true types. Here, we scan forward in the current block,
8084 // looking to see if this load dies before any side effects occur.
8085 // In such case, we can safely return the operand without making a copy.
8086 for (body[body_i..]) |body_inst| {
8087 switch (self.liveness.categorizeOperand(self.air, body_inst, inst)) {
8088 .none => continue,
8089 .write, .noret, .complex => break :elide,
8090 .tomb => return try self.resolveInst(ty_op.operand),
8091 }
8092 } else unreachable;
8093 }8121 }
8094 const ptr = try self.resolveInst(ty_op.operand);8122 return fg.load(ptr, ptr_ty);
8095 return self.load(ptr, ptr_ty);
8096 }8123 }
80978124
8098 fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {8125 fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
...@@ -9412,6 +9439,7 @@ pub const FuncGen = struct {...@@ -9412,6 +9439,7 @@ pub const FuncGen = struct {
9412 opt_llvm_ty: *llvm.Type,9439 opt_llvm_ty: *llvm.Type,
9413 opt_handle: *llvm.Value,9440 opt_handle: *llvm.Value,
9414 opt_ty: Type,9441 opt_ty: Type,
9442 can_elide_load: bool,
9415 ) !*llvm.Value {9443 ) !*llvm.Value {
9416 var buf: Type.Payload.ElemType = undefined;9444 var buf: Type.Payload.ElemType = undefined;
9417 const payload_ty = opt_ty.optionalChild(&buf);9445 const payload_ty = opt_ty.optionalChild(&buf);
...@@ -9420,11 +9448,14 @@ pub const FuncGen = struct {...@@ -9420,11 +9448,14 @@ pub const FuncGen = struct {
9420 // We have a pointer and we need to return a pointer to the first field.9448 // We have a pointer and we need to return a pointer to the first field.
9421 const payload_ptr = fg.builder.buildStructGEP(opt_llvm_ty, opt_handle, 0, "");9449 const payload_ptr = fg.builder.buildStructGEP(opt_llvm_ty, opt_handle, 0, "");
94229450
9423 if (isByRef(payload_ty)) {
9424 return payload_ptr;
9425 }
9426 const target = fg.dg.module.getTarget();9451 const target = fg.dg.module.getTarget();
9427 const payload_alignment = payload_ty.abiAlignment(target);9452 const payload_alignment = payload_ty.abiAlignment(target);
9453 if (isByRef(payload_ty)) {
9454 if (can_elide_load)
9455 return payload_ptr;
9456
9457 return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, false);
9458 }
9428 const payload_llvm_ty = try fg.dg.lowerType(payload_ty);9459 const payload_llvm_ty = try fg.dg.lowerType(payload_ty);
9429 const load_inst = fg.builder.buildLoad(payload_llvm_ty, payload_ptr, "");9460 const load_inst = fg.builder.buildLoad(payload_llvm_ty, payload_ptr, "");
9430 load_inst.setAlignment(payload_alignment);9461 load_inst.setAlignment(payload_alignment);
...@@ -9559,6 +9590,32 @@ pub const FuncGen = struct {...@@ -9559,6 +9590,32 @@ pub const FuncGen = struct {
9559 return self.llvmModule().getIntrinsicDeclaration(id, types.ptr, types.len);9590 return self.llvmModule().getIntrinsicDeclaration(id, types.ptr, types.len);
9560 }9591 }
95619592
9593 /// Load a by-ref type by constructing a new alloca and performing a memcpy.
9594 fn loadByRef(
9595 fg: *FuncGen,
9596 ptr: *llvm.Value,
9597 pointee_type: Type,
9598 ptr_alignment: u32,
9599 is_volatile: bool,
9600 ) !*llvm.Value {
9601 const pointee_llvm_ty = try fg.dg.lowerType(pointee_type);
9602 const target = fg.dg.module.getTarget();
9603 const result_align = @max(ptr_alignment, pointee_type.abiAlignment(target));
9604 const result_ptr = fg.buildAlloca(pointee_llvm_ty, result_align);
9605 const llvm_ptr_u8 = fg.context.intType(8).pointerType(0);
9606 const llvm_usize = fg.context.intType(Type.usize.intInfo(target).bits);
9607 const size_bytes = pointee_type.abiSize(target);
9608 _ = fg.builder.buildMemCpy(
9609 fg.builder.buildBitCast(result_ptr, llvm_ptr_u8, ""),
9610 result_align,
9611 fg.builder.buildBitCast(ptr, llvm_ptr_u8, ""),
9612 ptr_alignment,
9613 llvm_usize.constInt(size_bytes, .False),
9614 is_volatile,
9615 );
9616 return result_ptr;
9617 }
9618
9562 /// This function always performs a copy. For isByRef=true types, it creates a new9619 /// This function always performs a copy. For isByRef=true types, it creates a new
9563 /// alloca and copies the value into it, then returns the alloca instruction.9620 /// alloca and copies the value into it, then returns the alloca instruction.
9564 /// For isByRef=false types, it creates a load instruction and returns it.9621 /// For isByRef=false types, it creates a load instruction and returns it.
...@@ -9570,24 +9627,10 @@ pub const FuncGen = struct {...@@ -9570,24 +9627,10 @@ pub const FuncGen = struct {
9570 const ptr_alignment = info.alignment(target);9627 const ptr_alignment = info.alignment(target);
9571 const ptr_volatile = llvm.Bool.fromBool(ptr_ty.isVolatilePtr());9628 const ptr_volatile = llvm.Bool.fromBool(ptr_ty.isVolatilePtr());
9572 if (info.host_size == 0) {9629 if (info.host_size == 0) {
9573 const elem_llvm_ty = try self.dg.lowerType(info.pointee_type);
9574 if (isByRef(info.pointee_type)) {9630 if (isByRef(info.pointee_type)) {
9575 const result_align = info.pointee_type.abiAlignment(target);9631 return self.loadByRef(ptr, info.pointee_type, ptr_alignment, info.@"volatile");
9576 const max_align = @max(result_align, ptr_alignment);
9577 const result_ptr = self.buildAlloca(elem_llvm_ty, max_align);
9578 const llvm_ptr_u8 = self.context.intType(8).pointerType(0);
9579 const llvm_usize = self.context.intType(Type.usize.intInfo(target).bits);
9580 const size_bytes = info.pointee_type.abiSize(target);
9581 _ = self.builder.buildMemCpy(
9582 self.builder.buildBitCast(result_ptr, llvm_ptr_u8, ""),
9583 max_align,
9584 self.builder.buildBitCast(ptr, llvm_ptr_u8, ""),
9585 max_align,
9586 llvm_usize.constInt(size_bytes, .False),
9587 info.@"volatile",
9588 );
9589 return result_ptr;
9590 }9632 }
9633 const elem_llvm_ty = try self.dg.lowerType(info.pointee_type);
9591 const llvm_inst = self.builder.buildLoad(elem_llvm_ty, ptr, "");9634 const llvm_inst = self.builder.buildLoad(elem_llvm_ty, ptr, "");
9592 llvm_inst.setAlignment(ptr_alignment);9635 llvm_inst.setAlignment(ptr_alignment);
9593 llvm_inst.setVolatile(ptr_volatile);9636 llvm_inst.setVolatile(ptr_volatile);
test/behavior.zig+3
...@@ -104,7 +104,10 @@ test {...@@ -104,7 +104,10 @@ test {
104 _ = @import("behavior/bugs/12945.zig");104 _ = @import("behavior/bugs/12945.zig");
105 _ = @import("behavior/bugs/12972.zig");105 _ = @import("behavior/bugs/12972.zig");
106 _ = @import("behavior/bugs/12984.zig");106 _ = @import("behavior/bugs/12984.zig");
107 _ = @import("behavior/bugs/13064.zig");
108 _ = @import("behavior/bugs/13065.zig");
107 _ = @import("behavior/bugs/13068.zig");109 _ = @import("behavior/bugs/13068.zig");
110 _ = @import("behavior/bugs/13069.zig");
108 _ = @import("behavior/bugs/13112.zig");111 _ = @import("behavior/bugs/13112.zig");
109 _ = @import("behavior/bugs/13128.zig");112 _ = @import("behavior/bugs/13128.zig");
110 _ = @import("behavior/bugs/13164.zig");113 _ = @import("behavior/bugs/13164.zig");
test/behavior/bugs/13064.zig created+17
...@@ -0,0 +1,17 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const expect = std.testing.expect;
4
5test {
6 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
7 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
9
10 var x: [10][10]u32 = undefined;
11
12 x[0][1] = 0;
13 const a = x[0];
14 x[0][1] = 15;
15
16 try expect(a[1] == 0);
17}
test/behavior/bugs/13065.zig created+22
...@@ -0,0 +1,22 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const expect = std.testing.expect;
4
5const U = union(enum) {
6 array: [10]u32,
7 other: u32,
8};
9
10test {
11 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
12 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
13 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
14
15 var x = U{ .array = undefined };
16
17 x.array[1] = 0;
18 const a = x.array;
19 x.array[1] = 15;
20
21 try expect(a[1] == 0);
22}
test/behavior/bugs/13069.zig created+17
...@@ -0,0 +1,17 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const expect = std.testing.expect;
4
5test {
6 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
7 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
10
11 var opt_x: ?[3]f32 = [_]f32{0.0} ** 3;
12
13 const x = opt_x.?;
14 opt_x.?[0] = 15.0;
15
16 try expect(x[0] == 0.0);
17}