| ... | ... | @@ -2523,16 +2523,7 @@ pub const DeclGen = struct { |
| 2523 | 2523 | if (ptr_info.host_size != 0) { |
| 2524 | 2524 | return dg.context.intType(ptr_info.host_size * 8).pointerType(llvm_addrspace); |
| 2525 | 2525 | } |
| 2526 | | const elem_ty = ptr_info.pointee_type; |
| 2527 | | const lower_elem_ty = switch (elem_ty.zigTypeTag()) { |
| 2528 | | .Opaque, .Fn => true, |
| 2529 | | .Array => elem_ty.childType().hasRuntimeBitsIgnoreComptime(), |
| 2530 | | else => elem_ty.hasRuntimeBitsIgnoreComptime(), |
| 2531 | | }; |
| 2532 | | const llvm_elem_ty = if (lower_elem_ty) |
| 2533 | | try dg.lowerType(elem_ty) |
| 2534 | | else |
| 2535 | | dg.context.intType(8); |
| 2526 | const llvm_elem_ty = try dg.lowerPtrElemTy(ptr_info.pointee_type); |
| 2536 | 2527 | return llvm_elem_ty.pointerType(llvm_addrspace); |
| 2537 | 2528 | }, |
| 2538 | 2529 | .Opaque => switch (t.tag()) { |
| ... | ... | @@ -2917,6 +2908,23 @@ pub const DeclGen = struct { |
| 2917 | 2908 | ); |
| 2918 | 2909 | } |
| 2919 | 2910 | |
| 2911 | /// Use this instead of lowerType when you want to handle correctly the case of elem_ty |
| 2912 | /// being a zero bit type, but it should still be lowered as an i8 in such case. |
| 2913 | /// There are other similar cases handled here as well. |
| 2914 | fn lowerPtrElemTy(dg: *DeclGen, elem_ty: Type) Allocator.Error!*const llvm.Type { |
| 2915 | const lower_elem_ty = switch (elem_ty.zigTypeTag()) { |
| 2916 | .Opaque, .Fn => true, |
| 2917 | .Array => elem_ty.childType().hasRuntimeBitsIgnoreComptime(), |
| 2918 | else => elem_ty.hasRuntimeBitsIgnoreComptime(), |
| 2919 | }; |
| 2920 | const llvm_elem_ty = if (lower_elem_ty) |
| 2921 | try dg.lowerType(elem_ty) |
| 2922 | else |
| 2923 | dg.context.intType(8); |
| 2924 | |
| 2925 | return llvm_elem_ty; |
| 2926 | } |
| 2927 | |
| 2920 | 2928 | fn lowerValue(dg: *DeclGen, tv: TypedValue) Error!*const llvm.Value { |
| 2921 | 2929 | if (tv.val.isUndef()) { |
| 2922 | 2930 | const llvm_type = try dg.lowerType(tv.ty); |
| ... | ... | @@ -4674,8 +4682,8 @@ pub const FuncGen = struct { |
| 4674 | 4682 | _ = self.builder.buildBr(end_block); |
| 4675 | 4683 | |
| 4676 | 4684 | self.builder.positionBuilderAtEnd(both_pl_block); |
| 4677 | | const lhs_payload = self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty); |
| 4678 | | const rhs_payload = self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty); |
| 4685 | const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty); |
| 4686 | const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty); |
| 4679 | 4687 | const payload_cmp = try self.cmp(lhs_payload, rhs_payload, payload_ty, op); |
| 4680 | 4688 | _ = self.builder.buildBr(end_block); |
| 4681 | 4689 | const both_pl_block_end = self.builder.getInsertBlock(); |
| ... | ... | @@ -5133,7 +5141,8 @@ pub const FuncGen = struct { |
| 5133 | 5141 | |
| 5134 | 5142 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| 5135 | 5143 | const slice_ptr = try self.resolveInst(ty_op.operand); |
| 5136 | | const slice_llvm_ty = try self.dg.lowerType(self.air.typeOf(ty_op.operand).childType()); |
| 5144 | const slice_ptr_ty = self.air.typeOf(ty_op.operand); |
| 5145 | const slice_llvm_ty = try self.dg.lowerPtrElemTy(slice_ptr_ty.childType()); |
| 5137 | 5146 | |
| 5138 | 5147 | return self.builder.buildStructGEP(slice_llvm_ty, slice_ptr, index, ""); |
| 5139 | 5148 | } |
| ... | ... | @@ -5145,7 +5154,7 @@ pub const FuncGen = struct { |
| 5145 | 5154 | |
| 5146 | 5155 | const slice = try self.resolveInst(bin_op.lhs); |
| 5147 | 5156 | const index = try self.resolveInst(bin_op.rhs); |
| 5148 | | const llvm_elem_ty = try self.dg.lowerType(slice_ty.childType()); |
| 5157 | const llvm_elem_ty = try self.dg.lowerPtrElemTy(slice_ty.childType()); |
| 5149 | 5158 | const base_ptr = self.builder.buildExtractValue(slice, 0, ""); |
| 5150 | 5159 | const indices: [1]*const llvm.Value = .{index}; |
| 5151 | 5160 | const ptr = self.builder.buildInBoundsGEP(llvm_elem_ty, base_ptr, &indices, indices.len, ""); |
| ... | ... | @@ -5160,7 +5169,7 @@ pub const FuncGen = struct { |
| 5160 | 5169 | |
| 5161 | 5170 | const slice = try self.resolveInst(bin_op.lhs); |
| 5162 | 5171 | const index = try self.resolveInst(bin_op.rhs); |
| 5163 | | const llvm_elem_ty = try self.dg.lowerType(slice_ty.childType()); |
| 5172 | const llvm_elem_ty = try self.dg.lowerPtrElemTy(slice_ty.childType()); |
| 5164 | 5173 | const base_ptr = self.builder.buildExtractValue(slice, 0, ""); |
| 5165 | 5174 | const indices: [1]*const llvm.Value = .{index}; |
| 5166 | 5175 | return self.builder.buildInBoundsGEP(llvm_elem_ty, base_ptr, &indices, indices.len, ""); |
| ... | ... | @@ -5195,9 +5204,10 @@ pub const FuncGen = struct { |
| 5195 | 5204 | const ptr_ty = self.air.typeOf(bin_op.lhs); |
| 5196 | 5205 | if (!ptr_ty.isVolatilePtr() and self.liveness.isUnused(inst)) return null; |
| 5197 | 5206 | |
| 5198 | | const llvm_elem_ty = try self.dg.lowerType(ptr_ty.childType()); |
| 5207 | const llvm_elem_ty = try self.dg.lowerPtrElemTy(ptr_ty.childType()); |
| 5199 | 5208 | const base_ptr = try self.resolveInst(bin_op.lhs); |
| 5200 | 5209 | const rhs = try self.resolveInst(bin_op.rhs); |
| 5210 | // TODO: when we go fully opaque pointers in LLVM 16 we can remove this branch |
| 5201 | 5211 | const ptr = if (ptr_ty.isSinglePointer()) ptr: { |
| 5202 | 5212 | // If this is a single-item pointer to an array, we need another index in the GEP. |
| 5203 | 5213 | const indices: [2]*const llvm.Value = .{ self.context.intType(32).constNull(), rhs }; |
| ... | ... | @@ -5220,7 +5230,7 @@ pub const FuncGen = struct { |
| 5220 | 5230 | |
| 5221 | 5231 | const base_ptr = try self.resolveInst(bin_op.lhs); |
| 5222 | 5232 | const rhs = try self.resolveInst(bin_op.rhs); |
| 5223 | | const llvm_elem_ty = try self.dg.lowerType(elem_ty); |
| 5233 | const llvm_elem_ty = try self.dg.lowerPtrElemTy(elem_ty); |
| 5224 | 5234 | if (ptr_ty.isSinglePointer()) { |
| 5225 | 5235 | // If this is a single-item pointer to an array, we need another index in the GEP. |
| 5226 | 5236 | const indices: [2]*const llvm.Value = .{ self.context.intType(32).constNull(), rhs }; |
| ... | ... | @@ -5555,7 +5565,9 @@ pub const FuncGen = struct { |
| 5555 | 5565 | const max_param_count = inputs.len + outputs.len; |
| 5556 | 5566 | const llvm_param_types = try arena.alloc(*const llvm.Type, max_param_count); |
| 5557 | 5567 | const llvm_param_values = try arena.alloc(*const llvm.Value, max_param_count); |
| 5558 | | const llvm_param_attrs = try arena.alloc(bool, max_param_count); |
| 5568 | // This stores whether we need to add an elementtype attribute and |
| 5569 | // if so, the element type itself. |
| 5570 | const llvm_param_attrs = try arena.alloc(?*const llvm.Type, max_param_count); |
| 5559 | 5571 | const target = self.dg.module.getTarget(); |
| 5560 | 5572 | |
| 5561 | 5573 | var llvm_ret_i: usize = 0; |
| ... | ... | @@ -5573,7 +5585,7 @@ pub const FuncGen = struct { |
| 5573 | 5585 | // for the string, we still use the next u32 for the null terminator. |
| 5574 | 5586 | extra_i += (constraint.len + name.len + (2 + 3)) / 4; |
| 5575 | 5587 | |
| 5576 | | try llvm_constraints.ensureUnusedCapacity(self.gpa, constraint.len + 1); |
| 5588 | try llvm_constraints.ensureUnusedCapacity(self.gpa, constraint.len + 3); |
| 5577 | 5589 | if (total_i != 0) { |
| 5578 | 5590 | llvm_constraints.appendAssumeCapacity(','); |
| 5579 | 5591 | } |
| ... | ... | @@ -5582,8 +5594,10 @@ pub const FuncGen = struct { |
| 5582 | 5594 | // Pass any non-return outputs indirectly, if the constraint accepts a memory location |
| 5583 | 5595 | llvm_ret_indirect[i] = (output != .none) and constraintAllowsMemory(constraint); |
| 5584 | 5596 | if (output != .none) { |
| 5585 | | try llvm_constraints.ensureUnusedCapacity(self.gpa, llvm_constraints.capacity + 1); |
| 5586 | 5597 | const output_inst = try self.resolveInst(output); |
| 5598 | const output_ty = self.air.typeOf(output); |
| 5599 | assert(output_ty.zigTypeTag() == .Pointer); |
| 5600 | const elem_llvm_ty = try self.dg.lowerPtrElemTy(output_ty.childType()); |
| 5587 | 5601 | |
| 5588 | 5602 | if (llvm_ret_indirect[i]) { |
| 5589 | 5603 | // Pass the result by reference as an indirect output (e.g. "=*m") |
| ... | ... | @@ -5591,11 +5605,11 @@ pub const FuncGen = struct { |
| 5591 | 5605 | |
| 5592 | 5606 | llvm_param_values[llvm_param_i] = output_inst; |
| 5593 | 5607 | llvm_param_types[llvm_param_i] = output_inst.typeOf(); |
| 5594 | | llvm_param_attrs[llvm_param_i] = true; |
| 5608 | llvm_param_attrs[llvm_param_i] = elem_llvm_ty; |
| 5595 | 5609 | llvm_param_i += 1; |
| 5596 | 5610 | } else { |
| 5597 | 5611 | // Pass the result directly (e.g. "=r") |
| 5598 | | llvm_ret_types[llvm_ret_i] = output_inst.typeOf().getElementType(); |
| 5612 | llvm_ret_types[llvm_ret_i] = elem_llvm_ty; |
| 5599 | 5613 | llvm_ret_i += 1; |
| 5600 | 5614 | } |
| 5601 | 5615 | } else { |
| ... | ... | @@ -5633,7 +5647,9 @@ pub const FuncGen = struct { |
| 5633 | 5647 | |
| 5634 | 5648 | const arg_llvm_value = try self.resolveInst(input); |
| 5635 | 5649 | const arg_ty = self.air.typeOf(input); |
| 5650 | var llvm_elem_ty: ?*const llvm.Type = null; |
| 5636 | 5651 | if (isByRef(arg_ty)) { |
| 5652 | llvm_elem_ty = try self.dg.lowerPtrElemTy(arg_ty); |
| 5637 | 5653 | if (constraintAllowsMemory(constraint)) { |
| 5638 | 5654 | llvm_param_values[llvm_param_i] = arg_llvm_value; |
| 5639 | 5655 | llvm_param_types[llvm_param_i] = arg_llvm_value.typeOf(); |
| ... | ... | @@ -5677,7 +5693,12 @@ pub const FuncGen = struct { |
| 5677 | 5693 | |
| 5678 | 5694 | // In the case of indirect inputs, LLVM requires the callsite to have |
| 5679 | 5695 | // an elementtype(<ty>) attribute. |
| 5680 | | llvm_param_attrs[llvm_param_i] = constraint[0] == '*'; |
| 5696 | if (constraint[0] == '*') { |
| 5697 | llvm_param_attrs[llvm_param_i] = llvm_elem_ty orelse |
| 5698 | try self.dg.lowerPtrElemTy(arg_ty.childType()); |
| 5699 | } else { |
| 5700 | llvm_param_attrs[llvm_param_i] = null; |
| 5701 | } |
| 5681 | 5702 | |
| 5682 | 5703 | llvm_param_i += 1; |
| 5683 | 5704 | total_i += 1; |
| ... | ... | @@ -5812,10 +5833,9 @@ pub const FuncGen = struct { |
| 5812 | 5833 | .Auto, |
| 5813 | 5834 | "", |
| 5814 | 5835 | ); |
| 5815 | | for (llvm_param_attrs[0..param_count]) |need_elem_ty, i| { |
| 5816 | | if (need_elem_ty) { |
| 5817 | | const elem_ty = llvm_param_types[i].getElementType(); |
| 5818 | | llvm.setCallElemTypeAttr(call, i, elem_ty); |
| 5836 | for (llvm_param_attrs[0..param_count]) |llvm_elem_ty, i| { |
| 5837 | if (llvm_elem_ty) |llvm_ty| { |
| 5838 | llvm.setCallElemTypeAttr(call, i, llvm_ty); |
| 5819 | 5839 | } |
| 5820 | 5840 | } |
| 5821 | 5841 | |
| ... | ... | @@ -6558,7 +6578,7 @@ pub const FuncGen = struct { |
| 6558 | 6578 | const base_ptr = try self.resolveInst(bin_op.lhs); |
| 6559 | 6579 | const offset = try self.resolveInst(bin_op.rhs); |
| 6560 | 6580 | const ptr_ty = self.air.typeOf(bin_op.lhs); |
| 6561 | | const llvm_elem_ty = try self.dg.lowerType(ptr_ty.childType()); |
| 6581 | const llvm_elem_ty = try self.dg.lowerPtrElemTy(ptr_ty.childType()); |
| 6562 | 6582 | if (ptr_ty.ptrSize() == .One) { |
| 6563 | 6583 | // It's a pointer to an array, so according to LLVM we need an extra GEP index. |
| 6564 | 6584 | const indices: [2]*const llvm.Value = .{ |
| ... | ... | @@ -6580,7 +6600,7 @@ pub const FuncGen = struct { |
| 6580 | 6600 | const offset = try self.resolveInst(bin_op.rhs); |
| 6581 | 6601 | const negative_offset = self.builder.buildNeg(offset, ""); |
| 6582 | 6602 | const ptr_ty = self.air.typeOf(bin_op.lhs); |
| 6583 | | const llvm_elem_ty = try self.dg.lowerType(ptr_ty.childType()); |
| 6603 | const llvm_elem_ty = try self.dg.lowerPtrElemTy(ptr_ty.childType()); |
| 6584 | 6604 | if (ptr_ty.ptrSize() == .One) { |
| 6585 | 6605 | // It's a pointer to an array, so according to LLVM we need an extra GEP index. |
| 6586 | 6606 | const indices: [2]*const llvm.Value = .{ |
| ... | ... | @@ -7564,7 +7584,7 @@ pub const FuncGen = struct { |
| 7564 | 7584 | if (self.liveness.isUnused(inst)) return null; |
| 7565 | 7585 | |
| 7566 | 7586 | const llvm_i32 = self.context.intType(32); |
| 7567 | | const llvm_fn_name = "llvm.frameaddress.p0i8"; |
| 7587 | const llvm_fn_name = "llvm.frameaddress.p0"; |
| 7568 | 7588 | const llvm_fn = self.dg.object.llvm_module.getNamedFunction(llvm_fn_name) orelse blk: { |
| 7569 | 7589 | const llvm_p0i8 = self.context.intType(8).pointerType(0); |
| 7570 | 7590 | const param_types = [_]*const llvm.Type{llvm_i32}; |
| ... | ... | @@ -7694,20 +7714,31 @@ pub const FuncGen = struct { |
| 7694 | 7714 | const atomic_load = self.air.instructions.items(.data)[inst].atomic_load; |
| 7695 | 7715 | const ptr = try self.resolveInst(atomic_load.ptr); |
| 7696 | 7716 | const ptr_ty = self.air.typeOf(atomic_load.ptr); |
| 7697 | | if (!ptr_ty.isVolatilePtr() and self.liveness.isUnused(inst)) |
| 7717 | const ptr_info = ptr_ty.ptrInfo().data; |
| 7718 | if (!ptr_info.@"volatile" and self.liveness.isUnused(inst)) |
| 7719 | return null; |
| 7720 | const elem_ty = ptr_info.pointee_type; |
| 7721 | if (!elem_ty.hasRuntimeBitsIgnoreComptime()) |
| 7698 | 7722 | return null; |
| 7699 | 7723 | const ordering = toLlvmAtomicOrdering(atomic_load.order); |
| 7700 | | const operand_ty = ptr_ty.elemType(); |
| 7701 | | const opt_abi_ty = self.dg.getAtomicAbiType(operand_ty, false); |
| 7724 | const opt_abi_llvm_ty = self.dg.getAtomicAbiType(elem_ty, false); |
| 7725 | const target = self.dg.module.getTarget(); |
| 7726 | const ptr_alignment = ptr_info.alignment(target); |
| 7727 | const ptr_volatile = llvm.Bool.fromBool(ptr_info.@"volatile"); |
| 7728 | const elem_llvm_ty = try self.dg.lowerType(elem_ty); |
| 7702 | 7729 | |
| 7703 | | if (opt_abi_ty) |abi_ty| { |
| 7730 | if (opt_abi_llvm_ty) |abi_llvm_ty| { |
| 7704 | 7731 | // operand needs widening and truncating |
| 7705 | | const casted_ptr = self.builder.buildBitCast(ptr, abi_ty.pointerType(0), ""); |
| 7706 | | const load_inst = (try self.load(casted_ptr, ptr_ty)).?; |
| 7732 | const casted_ptr = self.builder.buildBitCast(ptr, abi_llvm_ty.pointerType(0), ""); |
| 7733 | const load_inst = self.builder.buildLoad(abi_llvm_ty, casted_ptr, ""); |
| 7734 | load_inst.setAlignment(ptr_alignment); |
| 7735 | load_inst.setVolatile(ptr_volatile); |
| 7707 | 7736 | load_inst.setOrdering(ordering); |
| 7708 | | return self.builder.buildTrunc(load_inst, try self.dg.lowerType(operand_ty), ""); |
| 7737 | return self.builder.buildTrunc(load_inst, elem_llvm_ty, ""); |
| 7709 | 7738 | } |
| 7710 | | const load_inst = (try self.load(ptr, ptr_ty)).?; |
| 7739 | const load_inst = self.builder.buildLoad(elem_llvm_ty, ptr, ""); |
| 7740 | load_inst.setAlignment(ptr_alignment); |
| 7741 | load_inst.setVolatile(ptr_volatile); |
| 7711 | 7742 | load_inst.setOrdering(ordering); |
| 7712 | 7743 | return load_inst; |
| 7713 | 7744 | } |
| ... | ... | @@ -8508,7 +8539,7 @@ pub const FuncGen = struct { |
| 8508 | 8539 | const llvm_ptr_u8 = llvm_u8.pointerType(0); |
| 8509 | 8540 | const llvm_u32 = self.context.intType(32); |
| 8510 | 8541 | |
| 8511 | | const llvm_fn_name = "llvm.prefetch.p0i8"; |
| 8542 | const llvm_fn_name = "llvm.prefetch.p0"; |
| 8512 | 8543 | const fn_val = self.dg.object.llvm_module.getNamedFunction(llvm_fn_name) orelse blk: { |
| 8513 | 8544 | // declare void @llvm.prefetch(i8*, i32, i32, i32) |
| 8514 | 8545 | const llvm_void = self.context.voidType(); |
| ... | ... | @@ -8660,7 +8691,7 @@ pub const FuncGen = struct { |
| 8660 | 8691 | opt_llvm_ty: *const llvm.Type, |
| 8661 | 8692 | opt_handle: *const llvm.Value, |
| 8662 | 8693 | opt_ty: Type, |
| 8663 | | ) *const llvm.Value { |
| 8694 | ) !*const llvm.Value { |
| 8664 | 8695 | var buf: Type.Payload.ElemType = undefined; |
| 8665 | 8696 | const payload_ty = opt_ty.optionalChild(&buf); |
| 8666 | 8697 | |
| ... | ... | @@ -8673,7 +8704,8 @@ pub const FuncGen = struct { |
| 8673 | 8704 | } |
| 8674 | 8705 | const target = fg.dg.module.getTarget(); |
| 8675 | 8706 | const payload_alignment = payload_ty.abiAlignment(target); |
| 8676 | | const load_inst = fg.builder.buildLoad(payload_ptr.getGEPResultElementType(), payload_ptr, ""); |
| 8707 | const payload_llvm_ty = try fg.dg.lowerType(payload_ty); |
| 8708 | const load_inst = fg.builder.buildLoad(payload_llvm_ty, payload_ptr, ""); |
| 8677 | 8709 | load_inst.setAlignment(payload_alignment); |
| 8678 | 8710 | return load_inst; |
| 8679 | 8711 | } |
| ... | ... | @@ -8758,7 +8790,7 @@ pub const FuncGen = struct { |
| 8758 | 8790 | return self.builder.buildBitCast(new_ptr, result_llvm_ty, ""); |
| 8759 | 8791 | }, |
| 8760 | 8792 | else => { |
| 8761 | | const struct_llvm_ty = try self.dg.lowerType(struct_ty); |
| 8793 | const struct_llvm_ty = try self.dg.lowerPtrElemTy(struct_ty); |
| 8762 | 8794 | |
| 8763 | 8795 | var ty_buf: Type.Payload.Pointer = undefined; |
| 8764 | 8796 | if (llvmFieldIndex(struct_ty, field_index, target, &ty_buf)) |llvm_field_index| { |
| ... | ... | @@ -8815,7 +8847,7 @@ pub const FuncGen = struct { |
| 8815 | 8847 | if (!info.pointee_type.hasRuntimeBitsIgnoreComptime()) return null; |
| 8816 | 8848 | |
| 8817 | 8849 | const target = self.dg.module.getTarget(); |
| 8818 | | const ptr_alignment = ptr_ty.ptrAlignment(target); |
| 8850 | const ptr_alignment = info.alignment(target); |
| 8819 | 8851 | const ptr_volatile = llvm.Bool.fromBool(ptr_ty.isVolatilePtr()); |
| 8820 | 8852 | if (info.host_size == 0) { |
| 8821 | 8853 | const elem_llvm_ty = try self.dg.lowerType(info.pointee_type); |
Andrew Kelley <andrew@ziglang.org> 2022-08-04 17:32:46-07:00