| ... | ... | @@ -1341,7 +1341,10 @@ pub const Object = struct { |
| 1341 | 1341 | try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); |
| 1342 | 1342 | } |
| 1343 | 1343 | } |
| 1344 | | if (param_ty.zigTypeTag(zcu) != .optional) { |
| 1344 | if (param_ty.zigTypeTag(zcu) != .optional and |
| 1345 | !ptr_info.flags.is_allowzero and |
| 1346 | ptr_info.flags.address_space == .generic) |
| 1347 | { |
| 1345 | 1348 | try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); |
| 1346 | 1349 | } |
| 1347 | 1350 | if (ptr_info.flags.is_const) { |
| ... | ... | @@ -1419,8 +1422,6 @@ pub const Object = struct { |
| 1419 | 1422 | } |
| 1420 | 1423 | } |
| 1421 | 1424 | |
| 1422 | | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 1423 | | |
| 1424 | 1425 | const file, const subprogram = if (!wip.strip) debug_info: { |
| 1425 | 1426 | const file = try o.getDebugFile(file_scope); |
| 1426 | 1427 | |
| ... | ... | @@ -1517,6 +1518,17 @@ pub const Object = struct { |
| 1517 | 1518 | else => |e| return e, |
| 1518 | 1519 | }; |
| 1519 | 1520 | |
| 1521 | // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole |
| 1522 | // function as considering null pointers valid so that LLVM's optimizers don't remove these |
| 1523 | // operations on the assumption that they're undefined behavior. |
| 1524 | if (fg.allowzero_access) { |
| 1525 | try attributes.addFnAttr(.null_pointer_is_valid, &o.builder); |
| 1526 | } else { |
| 1527 | _ = try attributes.removeFnAttr(.null_pointer_is_valid); |
| 1528 | } |
| 1529 | |
| 1530 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 1531 | |
| 1520 | 1532 | if (fg.fuzz) |*f| { |
| 1521 | 1533 | { |
| 1522 | 1534 | const array_llvm_ty = try o.builder.arrayType(f.pcs.items.len, .i8); |
| ... | ... | @@ -4349,7 +4361,10 @@ pub const Object = struct { |
| 4349 | 4361 | try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); |
| 4350 | 4362 | } |
| 4351 | 4363 | } |
| 4352 | | if (!param_ty.isPtrLikeOptional(zcu) and !ptr_info.flags.is_allowzero) { |
| 4364 | if (!param_ty.isPtrLikeOptional(zcu) and |
| 4365 | !ptr_info.flags.is_allowzero and |
| 4366 | ptr_info.flags.address_space == .generic) |
| 4367 | { |
| 4353 | 4368 | try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); |
| 4354 | 4369 | } |
| 4355 | 4370 | switch (fn_info.cc) { |
| ... | ... | @@ -4667,6 +4682,15 @@ pub const FuncGen = struct { |
| 4667 | 4682 | |
| 4668 | 4683 | disable_intrinsics: bool, |
| 4669 | 4684 | |
| 4685 | /// Have we seen loads or stores involving `allowzero` pointers? |
| 4686 | allowzero_access: bool = false, |
| 4687 | |
| 4688 | pub fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void { |
| 4689 | // LLVM already considers null pointers to be valid in non-generic address spaces, so avoid |
| 4690 | // pessimizing optimization for functions with accesses to such pointers. |
| 4691 | if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true; |
| 4692 | } |
| 4693 | |
| 4670 | 4694 | const Fuzz = struct { |
| 4671 | 4695 | counters_variable: Builder.Variable.Index, |
| 4672 | 4696 | pcs: std.ArrayListUnmanaged(Builder.Constant), |
| ... | ... | @@ -5392,7 +5416,10 @@ pub const FuncGen = struct { |
| 5392 | 5416 | try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); |
| 5393 | 5417 | } |
| 5394 | 5418 | } |
| 5395 | | if (param_ty.zigTypeTag(zcu) != .optional) { |
| 5419 | if (param_ty.zigTypeTag(zcu) != .optional and |
| 5420 | !ptr_info.flags.is_allowzero and |
| 5421 | ptr_info.flags.address_space == .generic) |
| 5422 | { |
| 5396 | 5423 | try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); |
| 5397 | 5424 | } |
| 5398 | 5425 | if (ptr_info.flags.is_const) { |
| ... | ... | @@ -5519,7 +5546,7 @@ pub const FuncGen = struct { |
| 5519 | 5546 | ptr_ty.ptrAlignment(zcu).toLlvm(), |
| 5520 | 5547 | try o.builder.intValue(.i8, 0xaa), |
| 5521 | 5548 | len, |
| 5522 | | if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, |
| 5549 | .normal, |
| 5523 | 5550 | self.disable_intrinsics, |
| 5524 | 5551 | ); |
| 5525 | 5552 | const owner_mod = self.ng.ownerModule(); |
| ... | ... | @@ -5754,8 +5781,8 @@ pub const FuncGen = struct { |
| 5754 | 5781 | // of optionals that are not pointers. |
| 5755 | 5782 | const is_by_ref = isByRef(scalar_ty, zcu); |
| 5756 | 5783 | const opt_llvm_ty = try o.lowerType(scalar_ty); |
| 5757 | | const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref); |
| 5758 | | const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref); |
| 5784 | const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal); |
| 5785 | const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal); |
| 5759 | 5786 | const llvm_i2 = try o.builder.intType(2); |
| 5760 | 5787 | const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, ""); |
| 5761 | 5788 | const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, ""); |
| ... | ... | @@ -6206,6 +6233,9 @@ pub const FuncGen = struct { |
| 6206 | 6233 | const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); |
| 6207 | 6234 | const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu); |
| 6208 | 6235 | const is_unused = self.liveness.isUnused(inst); |
| 6236 | |
| 6237 | self.maybeMarkAllowZeroAccess(self.typeOf(extra.data.ptr).ptrInfo(zcu)); |
| 6238 | |
| 6209 | 6239 | return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused, err_cold); |
| 6210 | 6240 | } |
| 6211 | 6241 | |
| ... | ... | @@ -6229,10 +6259,13 @@ pub const FuncGen = struct { |
| 6229 | 6259 | |
| 6230 | 6260 | if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { |
| 6231 | 6261 | const loaded = loaded: { |
| 6262 | const access_kind: Builder.MemoryAccessKind = |
| 6263 | if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 6264 | |
| 6232 | 6265 | if (!payload_has_bits) { |
| 6233 | 6266 | // TODO add alignment to this load |
| 6234 | 6267 | break :loaded if (operand_is_ptr) |
| 6235 | | try fg.wip.load(.normal, error_type, err_union, .default, "") |
| 6268 | try fg.wip.load(access_kind, error_type, err_union, .default, "") |
| 6236 | 6269 | else |
| 6237 | 6270 | err_union; |
| 6238 | 6271 | } |
| ... | ... | @@ -6242,7 +6275,7 @@ pub const FuncGen = struct { |
| 6242 | 6275 | try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, ""); |
| 6243 | 6276 | // TODO add alignment to this load |
| 6244 | 6277 | break :loaded try fg.wip.load( |
| 6245 | | .normal, |
| 6278 | if (operand_is_ptr) access_kind else .normal, |
| 6246 | 6279 | error_type, |
| 6247 | 6280 | err_field_ptr, |
| 6248 | 6281 | .default, |
| ... | ... | @@ -6751,10 +6784,14 @@ pub const FuncGen = struct { |
| 6751 | 6784 | if (self.canElideLoad(body_tail)) |
| 6752 | 6785 | return ptr; |
| 6753 | 6786 | |
| 6787 | self.maybeMarkAllowZeroAccess(slice_ty.ptrInfo(zcu)); |
| 6788 | |
| 6754 | 6789 | const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm(); |
| 6755 | | return self.loadByRef(ptr, elem_ty, elem_alignment, .normal); |
| 6790 | return self.loadByRef(ptr, elem_ty, elem_alignment, if (slice_ty.isVolatilePtr(zcu)) .@"volatile" else .normal); |
| 6756 | 6791 | } |
| 6757 | 6792 | |
| 6793 | self.maybeMarkAllowZeroAccess(slice_ty.ptrInfo(zcu)); |
| 6794 | |
| 6758 | 6795 | return self.load(ptr, slice_ty); |
| 6759 | 6796 | } |
| 6760 | 6797 | |
| ... | ... | @@ -6824,10 +6861,15 @@ pub const FuncGen = struct { |
| 6824 | 6861 | &.{rhs}, ""); |
| 6825 | 6862 | if (isByRef(elem_ty, zcu)) { |
| 6826 | 6863 | if (self.canElideLoad(body_tail)) return ptr; |
| 6864 | |
| 6865 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); |
| 6866 | |
| 6827 | 6867 | const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm(); |
| 6828 | | return self.loadByRef(ptr, elem_ty, elem_alignment, .normal); |
| 6868 | return self.loadByRef(ptr, elem_ty, elem_alignment, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal); |
| 6829 | 6869 | } |
| 6830 | 6870 | |
| 6871 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); |
| 6872 | |
| 6831 | 6873 | return self.load(ptr, ptr_ty); |
| 6832 | 6874 | } |
| 6833 | 6875 | |
| ... | ... | @@ -7235,6 +7277,8 @@ pub const FuncGen = struct { |
| 7235 | 7277 | }), |
| 7236 | 7278 | } |
| 7237 | 7279 | |
| 7280 | self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu)); |
| 7281 | |
| 7238 | 7282 | // Pass any non-return outputs indirectly, if the constraint accepts a memory location |
| 7239 | 7283 | is_indirect.* = constraintAllowsMemory(constraint); |
| 7240 | 7284 | if (is_indirect.*) { |
| ... | ... | @@ -7341,10 +7385,11 @@ pub const FuncGen = struct { |
| 7341 | 7385 | |
| 7342 | 7386 | // In the case of indirect inputs, LLVM requires the callsite to have |
| 7343 | 7387 | // an elementtype(<ty>) attribute. |
| 7344 | | llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') |
| 7345 | | try o.lowerPtrElemTy(if (is_by_ref) arg_ty else arg_ty.childType(zcu)) |
| 7346 | | else |
| 7347 | | .none; |
| 7388 | llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: { |
| 7389 | if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu)); |
| 7390 | |
| 7391 | break :blk try o.lowerPtrElemTy(if (is_by_ref) arg_ty else arg_ty.childType(zcu)); |
| 7392 | } else .none; |
| 7348 | 7393 | |
| 7349 | 7394 | llvm_param_i += 1; |
| 7350 | 7395 | total_i += 1; |
| ... | ... | @@ -7367,7 +7412,13 @@ pub const FuncGen = struct { |
| 7367 | 7412 | llvm_param_types[llvm_param_i] = llvm_rw_val.typeOfWip(&self.wip); |
| 7368 | 7413 | } else { |
| 7369 | 7414 | const alignment = rw_ty.abiAlignment(zcu).toLlvm(); |
| 7370 | | const loaded = try self.wip.load(.normal, llvm_elem_ty, llvm_rw_val, alignment, ""); |
| 7415 | const loaded = try self.wip.load( |
| 7416 | if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, |
| 7417 | llvm_elem_ty, |
| 7418 | llvm_rw_val, |
| 7419 | alignment, |
| 7420 | "", |
| 7421 | ); |
| 7371 | 7422 | llvm_param_values[llvm_param_i] = loaded; |
| 7372 | 7423 | llvm_param_types[llvm_param_i] = llvm_elem_ty; |
| 7373 | 7424 | } |
| ... | ... | @@ -7530,9 +7581,13 @@ pub const FuncGen = struct { |
| 7530 | 7581 | if (output != .none) { |
| 7531 | 7582 | const output_ptr = try self.resolveInst(output); |
| 7532 | 7583 | const output_ptr_ty = self.typeOf(output); |
| 7533 | | |
| 7534 | 7584 | const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm(); |
| 7535 | | _ = try self.wip.store(.normal, output_value, output_ptr, alignment); |
| 7585 | _ = try self.wip.store( |
| 7586 | if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, |
| 7587 | output_value, |
| 7588 | output_ptr, |
| 7589 | alignment, |
| 7590 | ); |
| 7536 | 7591 | } else { |
| 7537 | 7592 | ret_val = output_value; |
| 7538 | 7593 | } |
| ... | ... | @@ -7557,9 +7612,15 @@ pub const FuncGen = struct { |
| 7557 | 7612 | const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; |
| 7558 | 7613 | const optional_llvm_ty = try o.lowerType(optional_ty); |
| 7559 | 7614 | const payload_ty = optional_ty.optionalChild(zcu); |
| 7615 | |
| 7616 | const access_kind: Builder.MemoryAccessKind = |
| 7617 | if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 7618 | |
| 7619 | if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); |
| 7620 | |
| 7560 | 7621 | if (optional_ty.optionalReprIsPayload(zcu)) { |
| 7561 | 7622 | const loaded = if (operand_is_ptr) |
| 7562 | | try self.wip.load(.normal, optional_llvm_ty, operand, .default, "") |
| 7623 | try self.wip.load(access_kind, optional_llvm_ty, operand, .default, "") |
| 7563 | 7624 | else |
| 7564 | 7625 | operand; |
| 7565 | 7626 | if (payload_ty.isSlice(zcu)) { |
| ... | ... | @@ -7577,14 +7638,14 @@ pub const FuncGen = struct { |
| 7577 | 7638 | |
| 7578 | 7639 | if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 7579 | 7640 | const loaded = if (operand_is_ptr) |
| 7580 | | try self.wip.load(.normal, optional_llvm_ty, operand, .default, "") |
| 7641 | try self.wip.load(access_kind, optional_llvm_ty, operand, .default, "") |
| 7581 | 7642 | else |
| 7582 | 7643 | operand; |
| 7583 | 7644 | return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), ""); |
| 7584 | 7645 | } |
| 7585 | 7646 | |
| 7586 | 7647 | const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu); |
| 7587 | | return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref); |
| 7648 | return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind); |
| 7588 | 7649 | } |
| 7589 | 7650 | |
| 7590 | 7651 | fn airIsErr( |
| ... | ... | @@ -7604,6 +7665,9 @@ pub const FuncGen = struct { |
| 7604 | 7665 | const error_type = try o.errorIntType(); |
| 7605 | 7666 | const zero = try o.builder.intValue(error_type, 0); |
| 7606 | 7667 | |
| 7668 | const access_kind: Builder.MemoryAccessKind = |
| 7669 | if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 7670 | |
| 7607 | 7671 | if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { |
| 7608 | 7672 | const val: Builder.Constant = switch (cond) { |
| 7609 | 7673 | .eq => .true, // 0 == 0 |
| ... | ... | @@ -7613,9 +7677,11 @@ pub const FuncGen = struct { |
| 7613 | 7677 | return val.toValue(); |
| 7614 | 7678 | } |
| 7615 | 7679 | |
| 7680 | if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); |
| 7681 | |
| 7616 | 7682 | if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 7617 | 7683 | const loaded = if (operand_is_ptr) |
| 7618 | | try self.wip.load(.normal, try o.lowerType(err_union_ty), operand, .default, "") |
| 7684 | try self.wip.load(access_kind, try o.lowerType(err_union_ty), operand, .default, "") |
| 7619 | 7685 | else |
| 7620 | 7686 | operand; |
| 7621 | 7687 | return self.wip.icmp(cond, loaded, zero, ""); |
| ... | ... | @@ -7627,7 +7693,7 @@ pub const FuncGen = struct { |
| 7627 | 7693 | const err_union_llvm_ty = try o.lowerType(err_union_ty); |
| 7628 | 7694 | const err_field_ptr = |
| 7629 | 7695 | try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, ""); |
| 7630 | | break :loaded try self.wip.load(.normal, error_type, err_field_ptr, .default, ""); |
| 7696 | break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, .default, ""); |
| 7631 | 7697 | } else try self.wip.extractValue(operand, &.{err_field_index}, ""); |
| 7632 | 7698 | return self.wip.icmp(cond, loaded, zero, ""); |
| 7633 | 7699 | } |
| ... | ... | @@ -7660,12 +7726,19 @@ pub const FuncGen = struct { |
| 7660 | 7726 | const zcu = pt.zcu; |
| 7661 | 7727 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 7662 | 7728 | const operand = try self.resolveInst(ty_op.operand); |
| 7663 | | const optional_ty = self.typeOf(ty_op.operand).childType(zcu); |
| 7729 | const optional_ptr_ty = self.typeOf(ty_op.operand); |
| 7730 | const optional_ty = optional_ptr_ty.childType(zcu); |
| 7664 | 7731 | const payload_ty = optional_ty.optionalChild(zcu); |
| 7665 | 7732 | const non_null_bit = try o.builder.intValue(.i8, 1); |
| 7733 | |
| 7734 | const access_kind: Builder.MemoryAccessKind = |
| 7735 | if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 7736 | |
| 7666 | 7737 | if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 7738 | self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); |
| 7739 | |
| 7667 | 7740 | // We have a pointer to a i8. We need to set it to 1 and then return the same pointer. |
| 7668 | | _ = try self.wip.store(.normal, non_null_bit, operand, .default); |
| 7741 | _ = try self.wip.store(access_kind, non_null_bit, operand, .default); |
| 7669 | 7742 | return operand; |
| 7670 | 7743 | } |
| 7671 | 7744 | if (optional_ty.optionalReprIsPayload(zcu)) { |
| ... | ... | @@ -7677,8 +7750,11 @@ pub const FuncGen = struct { |
| 7677 | 7750 | // First set the non-null bit. |
| 7678 | 7751 | const optional_llvm_ty = try o.lowerType(optional_ty); |
| 7679 | 7752 | const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, ""); |
| 7753 | |
| 7754 | self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); |
| 7755 | |
| 7680 | 7756 | // TODO set alignment on this store |
| 7681 | | _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default); |
| 7757 | _ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default); |
| 7682 | 7758 | |
| 7683 | 7759 | // Then return the payload pointer (only if it's used). |
| 7684 | 7760 | if (self.liveness.isUnused(inst)) return .none; |
| ... | ... | @@ -7764,18 +7840,26 @@ pub const FuncGen = struct { |
| 7764 | 7840 | } |
| 7765 | 7841 | } |
| 7766 | 7842 | |
| 7843 | const access_kind: Builder.MemoryAccessKind = |
| 7844 | if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 7845 | |
| 7767 | 7846 | const payload_ty = err_union_ty.errorUnionPayload(zcu); |
| 7768 | 7847 | if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 7769 | 7848 | if (!operand_is_ptr) return operand; |
| 7770 | | return self.wip.load(.normal, error_type, operand, .default, ""); |
| 7849 | |
| 7850 | self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); |
| 7851 | |
| 7852 | return self.wip.load(access_kind, error_type, operand, .default, ""); |
| 7771 | 7853 | } |
| 7772 | 7854 | |
| 7773 | 7855 | const offset = try errUnionErrorOffset(payload_ty, pt); |
| 7774 | 7856 | |
| 7775 | 7857 | if (operand_is_ptr or isByRef(err_union_ty, zcu)) { |
| 7858 | if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); |
| 7859 | |
| 7776 | 7860 | const err_union_llvm_ty = try o.lowerType(err_union_ty); |
| 7777 | 7861 | const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); |
| 7778 | | return self.wip.load(.normal, error_type, err_field_ptr, .default, ""); |
| 7862 | return self.wip.load(access_kind, error_type, err_field_ptr, .default, ""); |
| 7779 | 7863 | } |
| 7780 | 7864 | |
| 7781 | 7865 | return self.wip.extractValue(operand, &.{offset}, ""); |
| ... | ... | @@ -7787,22 +7871,31 @@ pub const FuncGen = struct { |
| 7787 | 7871 | const zcu = pt.zcu; |
| 7788 | 7872 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 7789 | 7873 | const operand = try self.resolveInst(ty_op.operand); |
| 7790 | | const err_union_ty = self.typeOf(ty_op.operand).childType(zcu); |
| 7874 | const err_union_ptr_ty = self.typeOf(ty_op.operand); |
| 7875 | const err_union_ty = err_union_ptr_ty.childType(zcu); |
| 7791 | 7876 | |
| 7792 | 7877 | const payload_ty = err_union_ty.errorUnionPayload(zcu); |
| 7793 | 7878 | const non_error_val = try o.builder.intValue(try o.errorIntType(), 0); |
| 7879 | |
| 7880 | const access_kind: Builder.MemoryAccessKind = |
| 7881 | if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 7882 | |
| 7794 | 7883 | if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 7795 | | _ = try self.wip.store(.normal, non_error_val, operand, .default); |
| 7884 | self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); |
| 7885 | |
| 7886 | _ = try self.wip.store(access_kind, non_error_val, operand, .default); |
| 7796 | 7887 | return operand; |
| 7797 | 7888 | } |
| 7798 | 7889 | const err_union_llvm_ty = try o.lowerType(err_union_ty); |
| 7799 | 7890 | { |
| 7891 | self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); |
| 7892 | |
| 7800 | 7893 | const err_int_ty = try pt.errorIntType(); |
| 7801 | 7894 | const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm(); |
| 7802 | 7895 | const error_offset = try errUnionErrorOffset(payload_ty, pt); |
| 7803 | 7896 | // First set the non-error value. |
| 7804 | 7897 | const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, ""); |
| 7805 | | _ = try self.wip.store(.normal, non_error_val, non_null_ptr, error_alignment); |
| 7898 | _ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment); |
| 7806 | 7899 | } |
| 7807 | 7900 | // Then return the payload pointer (only if it is used). |
| 7808 | 7901 | if (self.liveness.isUnused(inst)) return .none; |
| ... | ... | @@ -8017,6 +8110,10 @@ pub const FuncGen = struct { |
| 8017 | 8110 | const index = try self.resolveInst(extra.lhs); |
| 8018 | 8111 | const operand = try self.resolveInst(extra.rhs); |
| 8019 | 8112 | |
| 8113 | self.maybeMarkAllowZeroAccess(vector_ptr_ty.ptrInfo(zcu)); |
| 8114 | |
| 8115 | // TODO: Emitting a load here is a violation of volatile semantics. Not fixable in general. |
| 8116 | // https://github.com/ziglang/zig/issues/18652#issuecomment-2452844908 |
| 8020 | 8117 | const access_kind: Builder.MemoryAccessKind = |
| 8021 | 8118 | if (vector_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 8022 | 8119 | const elem_llvm_ty = try o.lowerType(vector_ptr_ty.childType(zcu)); |
| ... | ... | @@ -9482,6 +9579,8 @@ pub const FuncGen = struct { |
| 9482 | 9579 | return .none; |
| 9483 | 9580 | } |
| 9484 | 9581 | |
| 9582 | self.maybeMarkAllowZeroAccess(ptr_info); |
| 9583 | |
| 9485 | 9584 | const len = try o.builder.intValue(try o.lowerType(Type.usize), operand_ty.abiSize(zcu)); |
| 9486 | 9585 | _ = try self.wip.callMemSet( |
| 9487 | 9586 | dest_ptr, |
| ... | ... | @@ -9497,6 +9596,8 @@ pub const FuncGen = struct { |
| 9497 | 9596 | return .none; |
| 9498 | 9597 | } |
| 9499 | 9598 | |
| 9599 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); |
| 9600 | |
| 9500 | 9601 | const src_operand = try self.resolveInst(bin_op.rhs); |
| 9501 | 9602 | try self.store(dest_ptr, ptr_ty, src_operand, .none); |
| 9502 | 9603 | return .none; |
| ... | ... | @@ -9539,6 +9640,9 @@ pub const FuncGen = struct { |
| 9539 | 9640 | if (!canElideLoad(fg, body_tail)) break :elide; |
| 9540 | 9641 | return ptr; |
| 9541 | 9642 | } |
| 9643 | |
| 9644 | fg.maybeMarkAllowZeroAccess(ptr_info); |
| 9645 | |
| 9542 | 9646 | return fg.load(ptr, ptr_ty); |
| 9543 | 9647 | } |
| 9544 | 9648 | |
| ... | ... | @@ -9598,6 +9702,8 @@ pub const FuncGen = struct { |
| 9598 | 9702 | new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, ""); |
| 9599 | 9703 | } |
| 9600 | 9704 | |
| 9705 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); |
| 9706 | |
| 9601 | 9707 | const result = try self.wip.cmpxchg( |
| 9602 | 9708 | kind, |
| 9603 | 9709 | if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, |
| ... | ... | @@ -9649,6 +9755,8 @@ pub const FuncGen = struct { |
| 9649 | 9755 | if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 9650 | 9756 | const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); |
| 9651 | 9757 | |
| 9758 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); |
| 9759 | |
| 9652 | 9760 | if (llvm_abi_ty != .none) { |
| 9653 | 9761 | // operand needs widening and truncating or bitcasting. |
| 9654 | 9762 | return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw( |
| ... | ... | @@ -9712,6 +9820,8 @@ pub const FuncGen = struct { |
| 9712 | 9820 | if (info.flags.is_volatile) .@"volatile" else .normal; |
| 9713 | 9821 | const elem_llvm_ty = try o.lowerType(elem_ty); |
| 9714 | 9822 | |
| 9823 | self.maybeMarkAllowZeroAccess(info); |
| 9824 | |
| 9715 | 9825 | if (llvm_abi_ty != .none) { |
| 9716 | 9826 | // operand needs widening and truncating |
| 9717 | 9827 | const loaded = try self.wip.loadAtomic( |
| ... | ... | @@ -9761,6 +9871,9 @@ pub const FuncGen = struct { |
| 9761 | 9871 | "", |
| 9762 | 9872 | ); |
| 9763 | 9873 | } |
| 9874 | |
| 9875 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); |
| 9876 | |
| 9764 | 9877 | try self.store(ptr, ptr_ty, element, ordering); |
| 9765 | 9878 | return .none; |
| 9766 | 9879 | } |
| ... | ... | @@ -9778,6 +9891,8 @@ pub const FuncGen = struct { |
| 9778 | 9891 | const access_kind: Builder.MemoryAccessKind = |
| 9779 | 9892 | if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 9780 | 9893 | |
| 9894 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); |
| 9895 | |
| 9781 | 9896 | if (try self.air.value(bin_op.rhs, pt)) |elem_val| { |
| 9782 | 9897 | if (elem_val.isUndefDeep(zcu)) { |
| 9783 | 9898 | // Even if safety is disabled, we still emit a memset to undefined since it conveys |
| ... | ... | @@ -9916,6 +10031,9 @@ pub const FuncGen = struct { |
| 9916 | 10031 | const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or |
| 9917 | 10032 | dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 9918 | 10033 | |
| 10034 | self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu)); |
| 10035 | self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu)); |
| 10036 | |
| 9919 | 10037 | _ = try self.wip.callMemCpy( |
| 9920 | 10038 | dest_ptr, |
| 9921 | 10039 | dest_ptr_ty.ptrAlignment(zcu).toLlvm(), |
| ... | ... | @@ -9959,20 +10077,27 @@ pub const FuncGen = struct { |
| 9959 | 10077 | const pt = o.pt; |
| 9960 | 10078 | const zcu = pt.zcu; |
| 9961 | 10079 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 9962 | | const un_ty = self.typeOf(bin_op.lhs).childType(zcu); |
| 10080 | const un_ptr_ty = self.typeOf(bin_op.lhs); |
| 10081 | const un_ty = un_ptr_ty.childType(zcu); |
| 9963 | 10082 | const layout = un_ty.unionGetLayout(zcu); |
| 9964 | 10083 | if (layout.tag_size == 0) return .none; |
| 10084 | |
| 10085 | const access_kind: Builder.MemoryAccessKind = |
| 10086 | if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; |
| 10087 | |
| 10088 | self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu)); |
| 10089 | |
| 9965 | 10090 | const union_ptr = try self.resolveInst(bin_op.lhs); |
| 9966 | 10091 | const new_tag = try self.resolveInst(bin_op.rhs); |
| 9967 | 10092 | if (layout.payload_size == 0) { |
| 9968 | 10093 | // TODO alignment on this store |
| 9969 | | _ = try self.wip.store(.normal, new_tag, union_ptr, .default); |
| 10094 | _ = try self.wip.store(access_kind, new_tag, union_ptr, .default); |
| 9970 | 10095 | return .none; |
| 9971 | 10096 | } |
| 9972 | 10097 | const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); |
| 9973 | 10098 | const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(un_ty), union_ptr, tag_index, ""); |
| 9974 | 10099 | // TODO alignment on this store |
| 9975 | | _ = try self.wip.store(.normal, new_tag, tag_field_ptr, .default); |
| 10100 | _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, .default); |
| 9976 | 10101 | return .none; |
| 9977 | 10102 | } |
| 9978 | 10103 | |
| ... | ... | @@ -10869,12 +10994,13 @@ pub const FuncGen = struct { |
| 10869 | 10994 | opt_llvm_ty: Builder.Type, |
| 10870 | 10995 | opt_handle: Builder.Value, |
| 10871 | 10996 | is_by_ref: bool, |
| 10997 | access_kind: Builder.MemoryAccessKind, |
| 10872 | 10998 | ) Allocator.Error!Builder.Value { |
| 10873 | 10999 | const o = self.ng.object; |
| 10874 | 11000 | const field = b: { |
| 10875 | 11001 | if (is_by_ref) { |
| 10876 | 11002 | const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, ""); |
| 10877 | | break :b try self.wip.load(.normal, .i8, field_ptr, .default, ""); |
| 11003 | break :b try self.wip.load(access_kind, .i8, field_ptr, .default, ""); |
| 10878 | 11004 | } |
| 10879 | 11005 | break :b try self.wip.extractValue(opt_handle, &.{1}, ""); |
| 10880 | 11006 | }; |
| ... | ... | @@ -11183,7 +11309,7 @@ pub const FuncGen = struct { |
| 11183 | 11309 | const vec_elem_ty = try o.lowerType(elem_ty); |
| 11184 | 11310 | const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); |
| 11185 | 11311 | |
| 11186 | | const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, ""); |
| 11312 | const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, ""); |
| 11187 | 11313 | |
| 11188 | 11314 | const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, ""); |
| 11189 | 11315 | |
| ... | ... | @@ -11196,7 +11322,7 @@ pub const FuncGen = struct { |
| 11196 | 11322 | const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8)); |
| 11197 | 11323 | assert(ordering == .none); |
| 11198 | 11324 | const containing_int = |
| 11199 | | try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, ""); |
| 11325 | try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, ""); |
| 11200 | 11326 | const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); |
| 11201 | 11327 | const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset); |
| 11202 | 11328 | // Convert to equally-sized integer type in order to perform the bit |