authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-05-04 14:06:54-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-05-04 17:34:16-07:00
log5b1c0d922c1061706ae1673333fcfb1d8fdd4602
tree4df989a3854adeaad833ad1c9970c972a77918b4
parent259f784241fb44e0a1b570daaf31ba2b9f164106

stage2: improve semantics of atomic operations

ZIR instructions updated: atomic_load, atomic_rmw, atomic_store, cmpxchg These no longer construct a pointer type as the result location. This solves a TODO that was preventing the pointer from possibly being volatile, as well as properly handling allowzero and addrspace. It also allows the pointer to be over-aligned, which may be needed depending on the target. As a consequence, the element type needs to be communicated in the ZIR. This is done by strategically making one of the operands be ResultLoc.ty instead of ResultLoc.coerced_ty if possible, or otherwise explicitly adding elem_type into the ZIR encoding, such as in the case of atomic_load. The pointer type of atomic operations is now checked in Sema by coercing it to an expected pointer type, that maybe over-aligned according to target requirements. Together with the previous commit, Zig now has smaller alignment for large integers, depending on the target, and yet still has type safety for atomic operations that specially require higher alignment.

6 files changed, 183 insertions(+), 132 deletions(-)

src/AstGen.zig+10-50
...@@ -7423,42 +7423,22 @@ fn builtinCall(...@@ -7423,42 +7423,22 @@ fn builtinCall(
7423 },7423 },
74247424
7425 .atomic_load => {7425 .atomic_load => {
7426 const int_type = try typeExpr(gz, scope, params[0]);7426 const result = try gz.addPlNode(.atomic_load, node, Zir.Inst.AtomicLoad{
7427 // TODO allow this pointer type to be volatile
7428 const ptr_type = try gz.add(.{ .tag = .ptr_type_simple, .data = .{
7429 .ptr_type_simple = .{
7430 .is_allowzero = false,
7431 .is_mutable = false,
7432 .is_volatile = false,
7433 .size = .One,
7434 .elem_type = int_type,
7435 },
7436 } });
7437 const result = try gz.addPlNode(.atomic_load, node, Zir.Inst.Bin{
7438 // zig fmt: off7427 // zig fmt: off
7439 .lhs = try expr(gz, scope, .{ .coerced_ty = ptr_type }, params[1]),7428 .elem_type = try typeExpr(gz, scope, params[0]),
7440 .rhs = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[2]),7429 .ptr = try expr (gz, scope, .none, params[1]),
7430 .ordering = try expr (gz, scope, .{ .coerced_ty = .atomic_order_type }, params[2]),
7441 // zig fmt: on7431 // zig fmt: on
7442 });7432 });
7443 return rvalue(gz, rl, result, node);7433 return rvalue(gz, rl, result, node);
7444 },7434 },
7445 .atomic_rmw => {7435 .atomic_rmw => {
7446 const int_type = try typeExpr(gz, scope, params[0]);7436 const int_type = try typeExpr(gz, scope, params[0]);
7447 // TODO allow this pointer type to be volatile
7448 const ptr_type = try gz.add(.{ .tag = .ptr_type_simple, .data = .{
7449 .ptr_type_simple = .{
7450 .is_allowzero = false,
7451 .is_mutable = true,
7452 .is_volatile = false,
7453 .size = .One,
7454 .elem_type = int_type,
7455 },
7456 } });
7457 const result = try gz.addPlNode(.atomic_rmw, node, Zir.Inst.AtomicRmw{7437 const result = try gz.addPlNode(.atomic_rmw, node, Zir.Inst.AtomicRmw{
7458 // zig fmt: off7438 // zig fmt: off
7459 .ptr = try expr(gz, scope, .{ .coerced_ty = ptr_type }, params[1]),7439 .ptr = try expr(gz, scope, .none, params[1]),
7460 .operation = try expr(gz, scope, .{ .coerced_ty = .atomic_rmw_op_type }, params[2]),7440 .operation = try expr(gz, scope, .{ .coerced_ty = .atomic_rmw_op_type }, params[2]),
7461 .operand = try expr(gz, scope, .{ .coerced_ty = int_type }, params[3]),7441 .operand = try expr(gz, scope, .{ .ty = int_type }, params[3]),
7462 .ordering = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[4]),7442 .ordering = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[4]),
7463 // zig fmt: on7443 // zig fmt: on
7464 });7444 });
...@@ -7466,20 +7446,10 @@ fn builtinCall(...@@ -7466,20 +7446,10 @@ fn builtinCall(
7466 },7446 },
7467 .atomic_store => {7447 .atomic_store => {
7468 const int_type = try typeExpr(gz, scope, params[0]);7448 const int_type = try typeExpr(gz, scope, params[0]);
7469 // TODO allow this pointer type to be volatile
7470 const ptr_type = try gz.add(.{ .tag = .ptr_type_simple, .data = .{
7471 .ptr_type_simple = .{
7472 .is_allowzero = false,
7473 .is_mutable = true,
7474 .is_volatile = false,
7475 .size = .One,
7476 .elem_type = int_type,
7477 },
7478 } });
7479 const result = try gz.addPlNode(.atomic_store, node, Zir.Inst.AtomicStore{7449 const result = try gz.addPlNode(.atomic_store, node, Zir.Inst.AtomicStore{
7480 // zig fmt: off7450 // zig fmt: off
7481 .ptr = try expr(gz, scope, .{ .coerced_ty = ptr_type }, params[1]),7451 .ptr = try expr(gz, scope, .none, params[1]),
7482 .operand = try expr(gz, scope, .{ .coerced_ty = int_type }, params[2]),7452 .operand = try expr(gz, scope, .{ .ty = int_type }, params[2]),
7483 .ordering = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[3]),7453 .ordering = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[3]),
7484 // zig fmt: on7454 // zig fmt: on
7485 });7455 });
...@@ -7684,20 +7654,10 @@ fn cmpxchg(...@@ -7684,20 +7654,10 @@ fn cmpxchg(
7684 tag: Zir.Inst.Tag,7654 tag: Zir.Inst.Tag,
7685) InnerError!Zir.Inst.Ref {7655) InnerError!Zir.Inst.Ref {
7686 const int_type = try typeExpr(gz, scope, params[0]);7656 const int_type = try typeExpr(gz, scope, params[0]);
7687 // TODO: allow this to be volatile
7688 const ptr_type = try gz.add(.{ .tag = .ptr_type_simple, .data = .{
7689 .ptr_type_simple = .{
7690 .is_allowzero = false,
7691 .is_mutable = true,
7692 .is_volatile = false,
7693 .size = .One,
7694 .elem_type = int_type,
7695 },
7696 } });
7697 const result = try gz.addPlNode(tag, node, Zir.Inst.Cmpxchg{7657 const result = try gz.addPlNode(tag, node, Zir.Inst.Cmpxchg{
7698 // zig fmt: off7658 // zig fmt: off
7699 .ptr = try expr(gz, scope, .{ .coerced_ty = ptr_type }, params[1]),7659 .ptr = try expr(gz, scope, .none, params[1]),
7700 .expected_value = try expr(gz, scope, .{ .coerced_ty = int_type }, params[2]),7660 .expected_value = try expr(gz, scope, .{ .ty = int_type }, params[2]),
7701 .new_value = try expr(gz, scope, .{ .coerced_ty = int_type }, params[3]),7661 .new_value = try expr(gz, scope, .{ .coerced_ty = int_type }, params[3]),
7702 .success_order = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[4]),7662 .success_order = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[4]),
7703 .failure_order = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[5]),7663 .failure_order = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[5]),
src/Sema.zig+87-75
...@@ -14715,51 +14715,64 @@ fn checkNumericType(...@@ -14715,51 +14715,64 @@ fn checkNumericType(
14715 }14715 }
14716}14716}
1471714717
14718fn checkAtomicOperandType(14718/// Returns the casted pointer.
14719fn checkAtomicPtrOperand(
14719 sema: *Sema,14720 sema: *Sema,
14720 block: *Block,14721 block: *Block,
14721 ty_src: LazySrcLoc,14722 elem_ty: Type,
14722 ty: Type,14723 elem_ty_src: LazySrcLoc,
14723) CompileError!void {14724 ptr: Air.Inst.Ref,
14724 var buffer: Type.Payload.Bits = undefined;14725 ptr_src: LazySrcLoc,
14726 ptr_const: bool,
14727) CompileError!Air.Inst.Ref {
14725 const target = sema.mod.getTarget();14728 const target = sema.mod.getTarget();
14726 const max_atomic_bits = target_util.largestAtomicBits(target);14729 var diag: target_util.AtomicPtrAlignmentDiagnostics = .{};
14727 const int_ty = switch (ty.zigTypeTag()) {14730 const alignment = target_util.atomicPtrAlignment(target, elem_ty, &diag) catch |err| switch (err) {
14728 .Int => ty,14731 error.FloatTooBig => return sema.fail(
14729 .Enum => ty.intTagType(&buffer),14732 block,
14730 .Float => {14733 elem_ty_src,
14731 const bit_count = ty.floatBits(target);14734 "expected {d}-bit float type or smaller; found {d}-bit float type",
14732 if (bit_count > max_atomic_bits) {14735 .{ diag.max_bits, diag.bits },
14733 return sema.fail(14736 ),
14734 block,14737 error.IntTooBig => return sema.fail(
14735 ty_src,14738 block,
14736 "expected {d}-bit float type or smaller; found {d}-bit float type",14739 elem_ty_src,
14737 .{ max_atomic_bits, bit_count },14740 "expected {d}-bit integer type or smaller; found {d}-bit integer type",
14738 );14741 .{ diag.max_bits, diag.bits },
14739 }14742 ),
14740 return;14743 error.BadType => return sema.fail(
14741 },14744 block,
14742 .Bool => return, // Will be treated as `u8`.14745 elem_ty_src,
14743 else => {14746 "expected bool, integer, float, enum, or pointer type; found {}",
14744 if (ty.isPtrAtRuntime()) return;14747 .{elem_ty.fmt(sema.mod)},
14748 ),
14749 };
1474514750
14746 return sema.fail(14751 var wanted_ptr_data: Type.Payload.Pointer.Data = .{
14747 block,14752 .pointee_type = elem_ty,
14748 ty_src,14753 .@"align" = alignment,
14749 "expected bool, integer, float, enum, or pointer type; found {}",14754 .@"addrspace" = .generic,
14750 .{ty.fmt(sema.mod)},14755 .mutable = !ptr_const,
14751 );14756 };
14757
14758 const ptr_ty = sema.typeOf(ptr);
14759 const ptr_data = switch (try ptr_ty.zigTypeTagOrPoison()) {
14760 .Pointer => ptr_ty.ptrInfo().data,
14761 else => {
14762 const wanted_ptr_ty = try Type.ptr(sema.arena, sema.mod, wanted_ptr_data);
14763 _ = try sema.coerce(block, wanted_ptr_ty, ptr, ptr_src);
14764 unreachable;
14752 },14765 },
14753 };14766 };
14754 const bit_count = int_ty.intInfo(target).bits;14767
14755 if (bit_count > max_atomic_bits) {14768 wanted_ptr_data.@"addrspace" = ptr_data.@"addrspace";
14756 return sema.fail(14769 wanted_ptr_data.@"allowzero" = ptr_data.@"allowzero";
14757 block,14770 wanted_ptr_data.@"volatile" = ptr_data.@"volatile";
14758 ty_src,14771
14759 "expected {d}-bit integer type or smaller; found {d}-bit integer type",14772 const wanted_ptr_ty = try Type.ptr(sema.arena, sema.mod, wanted_ptr_data);
14760 .{ max_atomic_bits, bit_count },14773 const casted_ptr = try sema.coerce(block, wanted_ptr_ty, ptr, ptr_src);
14761 );14774
14762 }14775 return casted_ptr;
14763}14776}
1476414777
14765fn checkPtrIsNotComptimeMutable(14778fn checkPtrIsNotComptimeMutable(
...@@ -15036,10 +15049,8 @@ fn zirCmpxchg(...@@ -15036,10 +15049,8 @@ fn zirCmpxchg(
15036 const success_order_src: LazySrcLoc = .{ .node_offset_builtin_call_arg4 = inst_data.src_node };15049 const success_order_src: LazySrcLoc = .{ .node_offset_builtin_call_arg4 = inst_data.src_node };
15037 const failure_order_src: LazySrcLoc = .{ .node_offset_builtin_call_arg5 = inst_data.src_node };15050 const failure_order_src: LazySrcLoc = .{ .node_offset_builtin_call_arg5 = inst_data.src_node };
15038 // zig fmt: on15051 // zig fmt: on
15039 const ptr = sema.resolveInst(extra.ptr);15052 const expected_value = sema.resolveInst(extra.expected_value);
15040 const ptr_ty = sema.typeOf(ptr);15053 const elem_ty = sema.typeOf(expected_value);
15041 const elem_ty = ptr_ty.elemType();
15042 try sema.checkAtomicOperandType(block, elem_ty_src, elem_ty);
15043 if (elem_ty.zigTypeTag() == .Float) {15054 if (elem_ty.zigTypeTag() == .Float) {
15044 return sema.fail(15055 return sema.fail(
15045 block,15056 block,
...@@ -15048,7 +15059,8 @@ fn zirCmpxchg(...@@ -15048,7 +15059,8 @@ fn zirCmpxchg(
15048 .{elem_ty.fmt(sema.mod)},15059 .{elem_ty.fmt(sema.mod)},
15049 );15060 );
15050 }15061 }
15051 const expected_value = try sema.coerce(block, elem_ty, sema.resolveInst(extra.expected_value), expected_src);15062 const uncasted_ptr = sema.resolveInst(extra.ptr);
15063 const ptr = try sema.checkAtomicPtrOperand(block, elem_ty, elem_ty_src, uncasted_ptr, ptr_src, false);
15052 const new_value = try sema.coerce(block, elem_ty, sema.resolveInst(extra.new_value), new_value_src);15064 const new_value = try sema.coerce(block, elem_ty, sema.resolveInst(extra.new_value), new_value_src);
15053 const success_order = try sema.resolveAtomicOrder(block, success_order_src, extra.success_order);15065 const success_order = try sema.resolveAtomicOrder(block, success_order_src, extra.success_order);
15054 const failure_order = try sema.resolveAtomicOrder(block, failure_order_src, extra.failure_order);15066 const failure_order = try sema.resolveAtomicOrder(block, failure_order_src, extra.failure_order);
...@@ -15081,6 +15093,7 @@ fn zirCmpxchg(...@@ -15081,6 +15093,7 @@ fn zirCmpxchg(
15081 // to become undef as well15093 // to become undef as well
15082 return sema.addConstUndef(result_ty);15094 return sema.addConstUndef(result_ty);
15083 }15095 }
15096 const ptr_ty = sema.typeOf(ptr);
15084 const stored_val = (try sema.pointerDeref(block, ptr_src, ptr_val, ptr_ty)) orelse break :rs ptr_src;15097 const stored_val = (try sema.pointerDeref(block, ptr_src, ptr_val, ptr_ty)) orelse break :rs ptr_src;
15085 const result_val = if (stored_val.eql(expected_val, elem_ty, sema.mod)) blk: {15098 const result_val = if (stored_val.eql(expected_val, elem_ty, sema.mod)) blk: {
15086 try sema.storePtr(block, src, ptr, new_value);15099 try sema.storePtr(block, src, ptr, new_value);
...@@ -15487,17 +15500,16 @@ fn zirSelect(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air....@@ -15487,17 +15500,16 @@ fn zirSelect(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.
1548715500
15488fn zirAtomicLoad(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {15501fn zirAtomicLoad(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
15489 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;15502 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
15490 const extra = sema.code.extraData(Zir.Inst.Bin, inst_data.payload_index).data;15503 const extra = sema.code.extraData(Zir.Inst.AtomicLoad, inst_data.payload_index).data;
15491 // zig fmt: off15504 // zig fmt: off
15492 const elem_ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };15505 const elem_ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
15493 const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };15506 const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
15494 const order_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };15507 const order_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };
15495 // zig fmt: on15508 // zig fmt: on
15496 const ptr = sema.resolveInst(extra.lhs);15509 const elem_ty = try sema.resolveType(block, elem_ty_src, extra.elem_type);
15497 const ptr_ty = sema.typeOf(ptr);15510 const uncasted_ptr = sema.resolveInst(extra.ptr);
15498 const elem_ty = ptr_ty.elemType();15511 const ptr = try sema.checkAtomicPtrOperand(block, elem_ty, elem_ty_src, uncasted_ptr, ptr_src, true);
15499 try sema.checkAtomicOperandType(block, elem_ty_src, elem_ty);15512 const order = try sema.resolveAtomicOrder(block, order_src, extra.ordering);
15500 const order = try sema.resolveAtomicOrder(block, order_src, extra.rhs);
1550115513
15502 switch (order) {15514 switch (order) {
15503 .Release, .AcqRel => {15515 .Release, .AcqRel => {
...@@ -15516,7 +15528,7 @@ fn zirAtomicLoad(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!...@@ -15516,7 +15528,7 @@ fn zirAtomicLoad(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!
15516 }15528 }
1551715529
15518 if (try sema.resolveDefinedValue(block, ptr_src, ptr)) |ptr_val| {15530 if (try sema.resolveDefinedValue(block, ptr_src, ptr)) |ptr_val| {
15519 if (try sema.pointerDeref(block, ptr_src, ptr_val, ptr_ty)) |elem_val| {15531 if (try sema.pointerDeref(block, ptr_src, ptr_val, sema.typeOf(ptr))) |elem_val| {
15520 return sema.addConstant(elem_ty, elem_val);15532 return sema.addConstant(elem_ty, elem_val);
15521 }15533 }
15522 }15534 }
...@@ -15536,19 +15548,19 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -15536,19 +15548,19 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
15536 const extra = sema.code.extraData(Zir.Inst.AtomicRmw, inst_data.payload_index).data;15548 const extra = sema.code.extraData(Zir.Inst.AtomicRmw, inst_data.payload_index).data;
15537 const src = inst_data.src();15549 const src = inst_data.src();
15538 // zig fmt: off15550 // zig fmt: off
15539 const operand_ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };15551 const elem_ty_src : LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
15540 const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };15552 const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
15541 const op_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };15553 const op_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };
15542 const operand_src : LazySrcLoc = .{ .node_offset_builtin_call_arg3 = inst_data.src_node };15554 const operand_src : LazySrcLoc = .{ .node_offset_builtin_call_arg3 = inst_data.src_node };
15543 const order_src : LazySrcLoc = .{ .node_offset_builtin_call_arg4 = inst_data.src_node };15555 const order_src : LazySrcLoc = .{ .node_offset_builtin_call_arg4 = inst_data.src_node };
15544 // zig fmt: on15556 // zig fmt: on
15545 const ptr = sema.resolveInst(extra.ptr);15557 const operand = sema.resolveInst(extra.operand);
15546 const ptr_ty = sema.typeOf(ptr);15558 const elem_ty = sema.typeOf(operand);
15547 const operand_ty = ptr_ty.elemType();15559 const uncasted_ptr = sema.resolveInst(extra.ptr);
15548 try sema.checkAtomicOperandType(block, operand_ty_src, operand_ty);15560 const ptr = try sema.checkAtomicPtrOperand(block, elem_ty, elem_ty_src, uncasted_ptr, ptr_src, false);
15549 const op = try sema.resolveAtomicRmwOp(block, op_src, extra.operation);15561 const op = try sema.resolveAtomicRmwOp(block, op_src, extra.operation);
1555015562
15551 switch (operand_ty.zigTypeTag()) {15563 switch (elem_ty.zigTypeTag()) {
15552 .Enum => if (op != .Xchg) {15564 .Enum => if (op != .Xchg) {
15553 return sema.fail(block, op_src, "@atomicRmw with enum only allowed with .Xchg", .{});15565 return sema.fail(block, op_src, "@atomicRmw with enum only allowed with .Xchg", .{});
15554 },15566 },
...@@ -15561,7 +15573,6 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -15561,7 +15573,6 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
15561 },15573 },
15562 else => {},15574 else => {},
15563 }15575 }
15564 const operand = try sema.coerce(block, operand_ty, sema.resolveInst(extra.operand), operand_src);
15565 const order = try sema.resolveAtomicOrder(block, order_src, extra.ordering);15576 const order = try sema.resolveAtomicOrder(block, order_src, extra.ordering);
1556615577
15567 if (order == .Unordered) {15578 if (order == .Unordered) {
...@@ -15569,8 +15580,8 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -15569,8 +15580,8 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
15569 }15580 }
1557015581
15571 // special case zero bit types15582 // special case zero bit types
15572 if (try sema.typeHasOnePossibleValue(block, operand_ty_src, operand_ty)) |val| {15583 if (try sema.typeHasOnePossibleValue(block, elem_ty_src, elem_ty)) |val| {
15573 return sema.addConstant(operand_ty, val);15584 return sema.addConstant(elem_ty, val);
15574 }15585 }
1557515586
15576 const runtime_src = if (try sema.resolveDefinedValue(block, ptr_src, ptr)) |ptr_val| rs: {15587 const runtime_src = if (try sema.resolveDefinedValue(block, ptr_src, ptr)) |ptr_val| rs: {
...@@ -15581,22 +15592,23 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -15581,22 +15592,23 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
15581 };15592 };
15582 if (ptr_val.isComptimeMutablePtr()) {15593 if (ptr_val.isComptimeMutablePtr()) {
15583 const target = sema.mod.getTarget();15594 const target = sema.mod.getTarget();
15595 const ptr_ty = sema.typeOf(ptr);
15584 const stored_val = (try sema.pointerDeref(block, ptr_src, ptr_val, ptr_ty)) orelse break :rs ptr_src;15596 const stored_val = (try sema.pointerDeref(block, ptr_src, ptr_val, ptr_ty)) orelse break :rs ptr_src;
15585 const new_val = switch (op) {15597 const new_val = switch (op) {
15586 // zig fmt: off15598 // zig fmt: off
15587 .Xchg => operand_val,15599 .Xchg => operand_val,
15588 .Add => try stored_val.numberAddWrap(operand_val, operand_ty, sema.arena, target),15600 .Add => try stored_val.numberAddWrap(operand_val, elem_ty, sema.arena, target),
15589 .Sub => try stored_val.numberSubWrap(operand_val, operand_ty, sema.arena, target),15601 .Sub => try stored_val.numberSubWrap(operand_val, elem_ty, sema.arena, target),
15590 .And => try stored_val.bitwiseAnd (operand_val, operand_ty, sema.arena, target),15602 .And => try stored_val.bitwiseAnd (operand_val, elem_ty, sema.arena, target),
15591 .Nand => try stored_val.bitwiseNand (operand_val, operand_ty, sema.arena, target),15603 .Nand => try stored_val.bitwiseNand (operand_val, elem_ty, sema.arena, target),
15592 .Or => try stored_val.bitwiseOr (operand_val, operand_ty, sema.arena, target),15604 .Or => try stored_val.bitwiseOr (operand_val, elem_ty, sema.arena, target),
15593 .Xor => try stored_val.bitwiseXor (operand_val, operand_ty, sema.arena, target),15605 .Xor => try stored_val.bitwiseXor (operand_val, elem_ty, sema.arena, target),
15594 .Max => stored_val.numberMax (operand_val, target),15606 .Max => stored_val.numberMax (operand_val, target),
15595 .Min => stored_val.numberMin (operand_val, target),15607 .Min => stored_val.numberMin (operand_val, target),
15596 // zig fmt: on15608 // zig fmt: on
15597 };15609 };
15598 try sema.storePtrVal(block, src, ptr_val, new_val, operand_ty);15610 try sema.storePtrVal(block, src, ptr_val, new_val, elem_ty);
15599 return sema.addConstant(operand_ty, stored_val);15611 return sema.addConstant(elem_ty, stored_val);
15600 } else break :rs ptr_src;15612 } else break :rs ptr_src;
15601 } else ptr_src;15613 } else ptr_src;
1560215614
...@@ -15620,15 +15632,15 @@ fn zirAtomicStore(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError...@@ -15620,15 +15632,15 @@ fn zirAtomicStore(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
15620 const extra = sema.code.extraData(Zir.Inst.AtomicStore, inst_data.payload_index).data;15632 const extra = sema.code.extraData(Zir.Inst.AtomicStore, inst_data.payload_index).data;
15621 const src = inst_data.src();15633 const src = inst_data.src();
15622 // zig fmt: off15634 // zig fmt: off
15623 const operand_ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };15635 const elem_ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
15624 const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };15636 const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
15625 const operand_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };15637 const operand_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };
15626 const order_src : LazySrcLoc = .{ .node_offset_builtin_call_arg3 = inst_data.src_node };15638 const order_src : LazySrcLoc = .{ .node_offset_builtin_call_arg3 = inst_data.src_node };
15627 // zig fmt: on15639 // zig fmt: on
15628 const ptr = sema.resolveInst(extra.ptr);15640 const operand = sema.resolveInst(extra.operand);
15629 const operand_ty = sema.typeOf(ptr).elemType();15641 const elem_ty = sema.typeOf(operand);
15630 try sema.checkAtomicOperandType(block, operand_ty_src, operand_ty);15642 const uncasted_ptr = sema.resolveInst(extra.ptr);
15631 const operand = try sema.coerce(block, operand_ty, sema.resolveInst(extra.operand), operand_src);15643 const ptr = try sema.checkAtomicPtrOperand(block, elem_ty, elem_ty_src, uncasted_ptr, ptr_src, false);
15632 const order = try sema.resolveAtomicOrder(block, order_src, extra.ordering);15644 const order = try sema.resolveAtomicOrder(block, order_src, extra.ordering);
1563315645
15634 const air_tag: Air.Inst.Tag = switch (order) {15646 const air_tag: Air.Inst.Tag = switch (order) {
src/Zir.zig+7-1
...@@ -903,7 +903,7 @@ pub const Inst = struct {...@@ -903,7 +903,7 @@ pub const Inst = struct {
903 /// Uses the `pl_node` union field with payload `Select`.903 /// Uses the `pl_node` union field with payload `Select`.
904 select,904 select,
905 /// Implements the `@atomicLoad` builtin.905 /// Implements the `@atomicLoad` builtin.
906 /// Uses the `pl_node` union field with payload `Bin`.906 /// Uses the `pl_node` union field with payload `AtomicLoad`.
907 atomic_load,907 atomic_load,
908 /// Implements the `@atomicRmw` builtin.908 /// Implements the `@atomicRmw` builtin.
909 /// Uses the `pl_node` union field with payload `AtomicRmw`.909 /// Uses the `pl_node` union field with payload `AtomicRmw`.
...@@ -3293,6 +3293,12 @@ pub const Inst = struct {...@@ -3293,6 +3293,12 @@ pub const Inst = struct {
3293 ordering: Ref,3293 ordering: Ref,
3294 };3294 };
32953295
3296 pub const AtomicLoad = struct {
3297 elem_type: Ref,
3298 ptr: Ref,
3299 ordering: Ref,
3300 };
3301
3296 pub const MulAdd = struct {3302 pub const MulAdd = struct {
3297 mulend1: Ref,3303 mulend1: Ref,
3298 mulend2: Ref,3304 mulend2: Ref,
src/print_zir.zig+14-1
...@@ -283,6 +283,7 @@ const Writer = struct {...@@ -283,6 +283,7 @@ const Writer = struct {
283 => try self.writeStructInit(stream, inst),283 => try self.writeStructInit(stream, inst),
284284
285 .cmpxchg_strong, .cmpxchg_weak => try self.writeCmpxchg(stream, inst),285 .cmpxchg_strong, .cmpxchg_weak => try self.writeCmpxchg(stream, inst),
286 .atomic_load => try self.writeAtomicLoad(stream, inst),
286 .atomic_store => try self.writeAtomicStore(stream, inst),287 .atomic_store => try self.writeAtomicStore(stream, inst),
287 .atomic_rmw => try self.writeAtomicRmw(stream, inst),288 .atomic_rmw => try self.writeAtomicRmw(stream, inst),
288 .memcpy => try self.writeMemcpy(stream, inst),289 .memcpy => try self.writeMemcpy(stream, inst),
...@@ -351,7 +352,6 @@ const Writer = struct {...@@ -351,7 +352,6 @@ const Writer = struct {
351 .offset_of,352 .offset_of,
352 .splat,353 .splat,
353 .reduce,354 .reduce,
354 .atomic_load,
355 .bitcast,355 .bitcast,
356 .vector_type,356 .vector_type,
357 .maximum,357 .maximum,
...@@ -929,6 +929,19 @@ const Writer = struct {...@@ -929,6 +929,19 @@ const Writer = struct {
929 try self.writeSrc(stream, inst_data.src());929 try self.writeSrc(stream, inst_data.src());
930 }930 }
931931
932 fn writeAtomicLoad(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void {
933 const inst_data = self.code.instructions.items(.data)[inst].pl_node;
934 const extra = self.code.extraData(Zir.Inst.AtomicLoad, inst_data.payload_index).data;
935
936 try self.writeInstRef(stream, extra.elem_type);
937 try stream.writeAll(", ");
938 try self.writeInstRef(stream, extra.ptr);
939 try stream.writeAll(", ");
940 try self.writeInstRef(stream, extra.ordering);
941 try stream.writeAll(") ");
942 try self.writeSrc(stream, inst_data.src());
943 }
944
932 fn writeAtomicStore(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void {945 fn writeAtomicStore(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void {
933 const inst_data = self.code.instructions.items(.data)[inst].pl_node;946 const inst_data = self.code.instructions.items(.data)[inst].pl_node;
934 const extra = self.code.extraData(Zir.Inst.AtomicStore, inst_data.payload_index).data;947 const extra = self.code.extraData(Zir.Inst.AtomicStore, inst_data.payload_index).data;
src/target.zig+64-4
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1const std = @import("std");1const std = @import("std");
2const llvm = @import("codegen/llvm/bindings.zig");2const llvm = @import("codegen/llvm/bindings.zig");
3const Type = @import("type.zig").Type;
34
4pub const ArchOsAbi = struct {5pub const ArchOsAbi = struct {
5 arch: std.Target.Cpu.Arch,6 arch: std.Target.Cpu.Arch,
...@@ -543,10 +544,28 @@ pub fn needUnwindTables(target: std.Target) bool {...@@ -543,10 +544,28 @@ pub fn needUnwindTables(target: std.Target) bool {
543 return target.os.tag == .windows;544 return target.os.tag == .windows;
544}545}
545546
546/// TODO this was ported from stage1 but it does not take into account CPU features,547pub const AtomicPtrAlignmentError = error{
547/// which can affect this value. Audit this!548 FloatTooBig,
548pub fn largestAtomicBits(target: std.Target) u32 {549 IntTooBig,
549 return switch (target.cpu.arch) {550 BadType,
551};
552
553pub const AtomicPtrAlignmentDiagnostics = struct {
554 bits: u16 = undefined,
555 max_bits: u16 = undefined,
556};
557
558/// If ABI alignment of `ty` is OK for atomic operations, returs 0.
559/// Otherwise returns the alignment required on a pointer for the target
560/// to perform atomic operations.
561pub fn atomicPtrAlignment(
562 target: std.Target,
563 ty: Type,
564 diags: *AtomicPtrAlignmentDiagnostics,
565) AtomicPtrAlignmentError!u32 {
566 // TODO this was ported from stage1 but it does not take into account CPU features,
567 // which can affect this value. Audit this!
568 const max_atomic_bits: u16 = switch (target.cpu.arch) {
550 .avr,569 .avr,
551 .msp430,570 .msp430,
552 .spu_2,571 .spu_2,
...@@ -611,6 +630,47 @@ pub fn largestAtomicBits(target: std.Target) u32 {...@@ -611,6 +630,47 @@ pub fn largestAtomicBits(target: std.Target) u32 {
611630
612 .x86_64 => 128,631 .x86_64 => 128,
613 };632 };
633
634 var buffer: Type.Payload.Bits = undefined;
635
636 const int_ty = switch (ty.zigTypeTag()) {
637 .Int => ty,
638 .Enum => ty.intTagType(&buffer),
639 .Float => {
640 const bit_count = ty.floatBits(target);
641 if (bit_count > max_atomic_bits) {
642 diags.* = .{
643 .bits = bit_count,
644 .max_bits = max_atomic_bits,
645 };
646 return error.FloatTooBig;
647 }
648 if (target.cpu.arch == .x86_64 and bit_count > 64) {
649 return 16;
650 }
651 return 0;
652 },
653 .Bool => return 0,
654 else => {
655 if (ty.isPtrAtRuntime()) return 0;
656 return error.BadType;
657 },
658 };
659
660 const bit_count = int_ty.intInfo(target).bits;
661 if (bit_count > max_atomic_bits) {
662 diags.* = .{
663 .bits = bit_count,
664 .max_bits = max_atomic_bits,
665 };
666 return error.IntTooBig;
667 }
668
669 if (target.cpu.arch == .x86_64 and bit_count > 64) {
670 return 16;
671 }
672
673 return 0;
614}674}
615675
616pub fn defaultAddressSpace(676pub fn defaultAddressSpace(
test/behavior/atomics.zig+1-1
...@@ -127,7 +127,7 @@ test "128-bit cmpxchg" {...@@ -127,7 +127,7 @@ test "128-bit cmpxchg" {
127}127}
128128
129fn test_u128_cmpxchg() !void {129fn test_u128_cmpxchg() !void {
130 var x: u128 = 1234;130 var x: u128 align(16) = 1234;
131 if (@cmpxchgWeak(u128, &x, 99, 5678, .SeqCst, .SeqCst)) |x1| {131 if (@cmpxchgWeak(u128, &x, 99, 5678, .SeqCst, .SeqCst)) |x1| {
132 try expect(x1 == 1234);132 try expect(x1 == 1234);
133 } else {133 } else {