authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-09-15 18:55:39-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-09-15 19:00:35-07:00
logb67d1810be3234c363ee2929ffcc91083bfb0ae5
treec4fda56637928dc68069dcdd28411d37cafd721b
parentf83a4b444c4c2fca7086fe6dbdaccc7b6d8f35ca

stage2: implement `@atomicRmw` and `@atomicLoad`

* langref: add some more "see also" links for atomics * Add the following AIR instructions - atomic_load - atomic_store_unordered - atomic_store_monotonic - atomic_store_release - atomic_store_seq_cst - atomic_rmw * Implement those AIR instructions in LLVM and C backends. * AstGen: make the `ty` result locations for `@atomicRmw`, `@atomicLoad`, and `@atomicStore` be `coerced_ty` to avoid unnecessary ZIR instructions when Sema will be doing the coercions redundantly. * Sema for `@atomicLoad` and `@atomicRmw` is done, however Sema for `@atomicStore` is not yet implemented. - comptime eval for `@atomicRmw` is not yet implemented. * Sema: flesh out `coerceInMemoryAllowed` a little bit more. It can now handle pointers.

13 files changed, 708 insertions(+), 76 deletions(-)

doc/langref.html.in+9-3
......@@ -7216,7 +7216,9 @@ fn func(y: *i32) void {
72167216 {#syntax#}T{#endsyntax#} must be a pointer, a {#syntax#}bool{#endsyntax#}, a float,
72177217 an integer or an enum.
72187218 </p>
7219 {#see_also|@atomicStore|@atomicRmw|@fence|@cmpxchgWeak|@cmpxchgStrong#}
72197220 {#header_close#}
7221
72207222 {#header_open|@atomicRmw#}
72217223 <pre>{#syntax#}@atomicRmw(comptime T: type, ptr: *T, comptime op: builtin.AtomicRmwOp, operand: T, comptime ordering: builtin.AtomicOrder) T{#endsyntax#}</pre>
72227224 <p>
......@@ -7242,7 +7244,9 @@ fn func(y: *i32) void {
72427244 <li>{#syntax#}.Max{#endsyntax#} - stores the operand if it is larger. Supports integers and floats.</li>
72437245 <li>{#syntax#}.Min{#endsyntax#} - stores the operand if it is smaller. Supports integers and floats.</li>
72447246 </ul>
7247 {#see_also|@atomicStore|@atomicLoad|@fence|@cmpxchgWeak|@cmpxchgStrong#}
72457248 {#header_close#}
7249
72467250 {#header_open|@atomicStore#}
72477251 <pre>{#syntax#}@atomicStore(comptime T: type, ptr: *T, value: T, comptime ordering: builtin.AtomicOrder) void{#endsyntax#}</pre>
72487252 <p>
......@@ -7252,6 +7256,7 @@ fn func(y: *i32) void {
72527256 {#syntax#}T{#endsyntax#} must be a pointer, a {#syntax#}bool{#endsyntax#}, a float,
72537257 an integer or an enum.
72547258 </p>
7259 {#see_also|@atomicLoad|@atomicRmw|@fence|@cmpxchgWeak|@cmpxchgStrong#}
72557260 {#header_close#}
72567261
72577262 {#header_open|@bitCast#}
......@@ -7540,8 +7545,9 @@ fn cmpxchgStrongButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_v
75407545 an integer or an enum.
75417546 </p>
75427547 <p>{#syntax#}@typeInfo(@TypeOf(ptr)).Pointer.alignment{#endsyntax#} must be {#syntax#}>= @sizeOf(T).{#endsyntax#}</p>
7543 {#see_also|Compile Variables|cmpxchgWeak#}
7548 {#see_also|@atomicStore|@atomicLoad|@atomicRmw|@fence|@cmpxchgWeak#}
75447549 {#header_close#}
7550
75457551 {#header_open|@cmpxchgWeak#}
75467552 <pre>{#syntax#}@cmpxchgWeak(comptime T: type, ptr: *T, expected_value: T, new_value: T, success_order: AtomicOrder, fail_order: AtomicOrder) ?T{#endsyntax#}</pre>
75477553 <p>
......@@ -7569,7 +7575,7 @@ fn cmpxchgWeakButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_val
75697575 an integer or an enum.
75707576 </p>
75717577 <p>{#syntax#}@typeInfo(@TypeOf(ptr)).Pointer.alignment{#endsyntax#} must be {#syntax#}>= @sizeOf(T).{#endsyntax#}</p>
7572 {#see_also|Compile Variables|cmpxchgStrong#}
7578 {#see_also|@atomicStore|@atomicLoad|@atomicRmw|@fence|@cmpxchgStrong#}
75737579 {#header_close#}
75747580
75757581 {#header_open|@compileError#}
......@@ -7849,7 +7855,7 @@ export fn @"A function name that is a complete sentence."() void {}
78497855 <p>
78507856 {#syntax#}AtomicOrder{#endsyntax#} can be found with {#syntax#}@import("std").builtin.AtomicOrder{#endsyntax#}.
78517857 </p>
7852 {#see_also|Compile Variables#}
7858 {#see_also|@atomicStore|@atomicLoad|@atomicRmw|@cmpxchgWeak|@cmpxchgStrong#}
78537859 {#header_close#}
78547860
78557861 {#header_open|@field#}
src/Air.zig+56-4
......@@ -127,14 +127,11 @@ pub const Inst = struct {
127127 /// Lowers to a hardware trap instruction, or the next best thing.
128128 /// Result type is always void.
129129 breakpoint,
130 /// Lowers to a memory fence instruction.
131 /// Result type is always void.
132 /// Uses the `fence` field.
133 fence,
134130 /// Function call.
135131 /// Result type is the return type of the function being called.
136132 /// Uses the `pl_op` field with the `Call` payload. operand is the callee.
137133 call,
134
138135 /// `<`. Result type is always bool.
139136 /// Uses the `bin_op` field.
140137 cmp_lt,
......@@ -153,6 +150,7 @@ pub const Inst = struct {
153150 /// `!=`. Result type is always bool.
154151 /// Uses the `bin_op` field.
155152 cmp_neq,
153
156154 /// Conditional branch.
157155 /// Result type is always noreturn; no instructions in a block follow this one.
158156 /// Uses the `pl_op` field. Operand is the condition. Payload is `CondBr`.
......@@ -313,10 +311,33 @@ pub const Inst = struct {
313311 /// Given a pointer to an array, return a slice.
314312 /// Uses the `ty_op` field.
315313 array_to_slice,
314
316315 /// Uses the `ty_pl` field with payload `Cmpxchg`.
317316 cmpxchg_weak,
318317 /// Uses the `ty_pl` field with payload `Cmpxchg`.
319318 cmpxchg_strong,
319 /// Lowers to a memory fence instruction.
320 /// Result type is always void.
321 /// Uses the `fence` field.
322 fence,
323 /// Atomically load from a pointer.
324 /// Result type is the element type of the pointer.
325 /// Uses the `atomic_load` field.
326 atomic_load,
327 /// Atomically store through a pointer.
328 /// Result type is always `void`.
329 /// Uses the `bin_op` field. LHS is pointer, RHS is element.
330 atomic_store_unordered,
331 /// Same as `atomic_store_unordered` but with `AtomicOrder.Monotonic`.
332 atomic_store_monotonic,
333 /// Same as `atomic_store_unordered` but with `AtomicOrder.Release`.
334 atomic_store_release,
335 /// Same as `atomic_store_unordered` but with `AtomicOrder.SeqCst`.
336 atomic_store_seq_cst,
337 /// Atomically read-modify-write via a pointer.
338 /// Result type is the element type of the pointer.
339 /// Uses the `pl_op` field with payload `AtomicRmw`. Operand is `ptr`.
340 atomic_rmw,
320341
321342 pub fn fromCmpOp(op: std.math.CompareOperator) Tag {
322343 return switch (op) {
......@@ -385,6 +406,10 @@ pub const Inst = struct {
385406 column: u32,
386407 },
387408 fence: std.builtin.AtomicOrder,
409 atomic_load: struct {
410 ptr: Ref,
411 order: std.builtin.AtomicOrder,
412 },
388413
389414 // Make sure we don't accidentally add a field to make this union
390415 // bigger than expected. Note that in Debug builds, Zig is allowed
......@@ -469,6 +494,21 @@ pub const Cmpxchg = struct {
469494 }
470495};
471496
497pub const AtomicRmw = struct {
498 operand: Inst.Ref,
499 /// 0b00000000000000000000000000000XXX - ordering
500 /// 0b0000000000000000000000000XXXX000 - op
501 flags: u32,
502
503 pub fn ordering(self: AtomicRmw) std.builtin.AtomicOrder {
504 return @intToEnum(std.builtin.AtomicOrder, @truncate(u3, self.flags));
505 }
506
507 pub fn op(self: AtomicRmw) std.builtin.AtomicRmwOp {
508 return @intToEnum(std.builtin.AtomicRmwOp, @truncate(u4, self.flags >> 3));
509 }
510};
511
472512pub fn getMainBody(air: Air) []const Air.Inst.Index {
473513 const body_index = air.extra[@enumToInt(ExtraIndex.main_block)];
474514 const extra = air.extraData(Block, body_index);
......@@ -572,6 +612,10 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
572612 .dbg_stmt,
573613 .store,
574614 .fence,
615 .atomic_store_unordered,
616 .atomic_store_monotonic,
617 .atomic_store_release,
618 .atomic_store_seq_cst,
575619 => return Type.initTag(.void),
576620
577621 .ptrtoint,
......@@ -594,6 +638,14 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
594638 const inner_ptr_ty = outer_ptr_ty.elemType();
595639 return inner_ptr_ty.elemType();
596640 },
641 .atomic_load => {
642 const ptr_ty = air.typeOf(datas[inst].atomic_load.ptr);
643 return ptr_ty.elemType();
644 },
645 .atomic_rmw => {
646 const ptr_ty = air.typeOf(datas[inst].pl_op.operand);
647 return ptr_ty.elemType();
648 },
597649 }
598650}
599651
src/AstGen.zig+19-20
......@@ -7316,6 +7316,7 @@ fn builtinCall(
73167316
73177317 .atomic_load => {
73187318 const int_type = try typeExpr(gz, scope, params[0]);
7319 // TODO allow this pointer type to be volatile
73197320 const ptr_type = try gz.add(.{ .tag = .ptr_type_simple, .data = .{
73207321 .ptr_type_simple = .{
73217322 .is_allowzero = false,
......@@ -7325,16 +7326,17 @@ fn builtinCall(
73257326 .elem_type = int_type,
73267327 },
73277328 } });
7328 const ptr = try expr(gz, scope, .{ .ty = ptr_type }, params[1]);
7329 const ordering = try expr(gz, scope, .{ .ty = .atomic_order_type }, params[2]);
73307329 const result = try gz.addPlNode(.atomic_load, node, Zir.Inst.Bin{
7331 .lhs = ptr,
7332 .rhs = ordering,
7330 // zig fmt: off
7331 .lhs = try expr(gz, scope, .{ .coerced_ty = ptr_type }, params[1]),
7332 .rhs = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[2]),
7333 // zig fmt: on
73337334 });
73347335 return rvalue(gz, rl, result, node);
73357336 },
73367337 .atomic_rmw => {
73377338 const int_type = try typeExpr(gz, scope, params[0]);
7339 // TODO allow this pointer type to be volatile
73387340 const ptr_type = try gz.add(.{ .tag = .ptr_type_simple, .data = .{
73397341 .ptr_type_simple = .{
73407342 .is_allowzero = false,
......@@ -7344,20 +7346,19 @@ fn builtinCall(
73447346 .elem_type = int_type,
73457347 },
73467348 } });
7347 const ptr = try expr(gz, scope, .{ .ty = ptr_type }, params[1]);
7348 const operation = try expr(gz, scope, .{ .ty = .atomic_rmw_op_type }, params[2]);
7349 const operand = try expr(gz, scope, .{ .ty = int_type }, params[3]);
7350 const ordering = try expr(gz, scope, .{ .ty = .atomic_order_type }, params[4]);
73517349 const result = try gz.addPlNode(.atomic_rmw, node, Zir.Inst.AtomicRmw{
7352 .ptr = ptr,
7353 .operation = operation,
7354 .operand = operand,
7355 .ordering = ordering,
7350 // zig fmt: off
7351 .ptr = try expr(gz, scope, .{ .coerced_ty = ptr_type }, params[1]),
7352 .operation = try expr(gz, scope, .{ .coerced_ty = .atomic_rmw_op_type }, params[2]),
7353 .operand = try expr(gz, scope, .{ .coerced_ty = int_type }, params[3]),
7354 .ordering = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[4]),
7355 // zig fmt: on
73567356 });
73577357 return rvalue(gz, rl, result, node);
73587358 },
73597359 .atomic_store => {
73607360 const int_type = try typeExpr(gz, scope, params[0]);
7361 // TODO allow this pointer type to be volatile
73617362 const ptr_type = try gz.add(.{ .tag = .ptr_type_simple, .data = .{
73627363 .ptr_type_simple = .{
73637364 .is_allowzero = false,
......@@ -7367,13 +7368,12 @@ fn builtinCall(
73677368 .elem_type = int_type,
73687369 },
73697370 } });
7370 const ptr = try expr(gz, scope, .{ .ty = ptr_type }, params[1]);
7371 const operand = try expr(gz, scope, .{ .ty = int_type }, params[2]);
7372 const ordering = try expr(gz, scope, .{ .ty = .atomic_order_type }, params[3]);
73737371 const result = try gz.addPlNode(.atomic_store, node, Zir.Inst.AtomicStore{
7374 .ptr = ptr,
7375 .operand = operand,
7376 .ordering = ordering,
7372 // zig fmt: off
7373 .ptr = try expr(gz, scope, .{ .coerced_ty = ptr_type }, params[1]),
7374 .operand = try expr(gz, scope, .{ .coerced_ty = int_type }, params[2]),
7375 .ordering = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[3]),
7376 // zig fmt: on
73777377 });
73787378 return rvalue(gz, rl, result, node);
73797379 },
......@@ -7456,12 +7456,11 @@ fn builtinCall(
74567456 },
74577457 .Vector => {
74587458 const result = try gz.addPlNode(.vector_type, node, Zir.Inst.Bin{
7459 .lhs = try comptimeExpr(gz, scope, .{.ty = .u32_type}, params[0]),
7459 .lhs = try comptimeExpr(gz, scope, .{ .ty = .u32_type }, params[0]),
74607460 .rhs = try typeExpr(gz, scope, params[1]),
74617461 });
74627462 return rvalue(gz, rl, result, node);
74637463 },
7464
74657464 }
74667465 // zig fmt: on
74677466}
src/Liveness.zig+13
......@@ -252,6 +252,10 @@ fn analyzeInst(
252252 .ptr_ptr_elem_val,
253253 .shl,
254254 .shr,
255 .atomic_store_unordered,
256 .atomic_store_monotonic,
257 .atomic_store_release,
258 .atomic_store_seq_cst,
255259 => {
256260 const o = inst_datas[inst].bin_op;
257261 return trackOperands(a, new_set, inst, main_tomb, .{ o.lhs, o.rhs, .none });
......@@ -345,6 +349,15 @@ fn analyzeInst(
345349 const extra = a.air.extraData(Air.Cmpxchg, inst_datas[inst].ty_pl.payload).data;
346350 return trackOperands(a, new_set, inst, main_tomb, .{ extra.ptr, extra.expected_value, extra.new_value });
347351 },
352 .atomic_load => {
353 const ptr = inst_datas[inst].atomic_load.ptr;
354 return trackOperands(a, new_set, inst, main_tomb, .{ ptr, .none, .none });
355 },
356 .atomic_rmw => {
357 const pl_op = inst_datas[inst].pl_op;
358 const extra = a.air.extraData(Air.AtomicRmw, pl_op.payload).data;
359 return trackOperands(a, new_set, inst, main_tomb, .{ pl_op.operand, extra.operand, .none });
360 },
348361 .br => {
349362 const br = inst_datas[inst].br;
350363 return trackOperands(a, new_set, inst, main_tomb, .{ br.operand, .none, .none });
src/Sema.zig+185-7
......@@ -7549,6 +7549,19 @@ fn resolveAtomicOrder(
75497549 return val.toEnum(std.builtin.AtomicOrder);
75507550}
75517551
7552fn resolveAtomicRmwOp(
7553 sema: *Sema,
7554 block: *Scope.Block,
7555 src: LazySrcLoc,
7556 zir_ref: Zir.Inst.Ref,
7557) CompileError!std.builtin.AtomicRmwOp {
7558 const atomic_rmw_op_ty = try sema.getBuiltinType(block, src, "AtomicRmwOp");
7559 const air_ref = sema.resolveInst(zir_ref);
7560 const coerced = try sema.coerce(block, atomic_rmw_op_ty, air_ref, src);
7561 const val = try sema.resolveConstValue(block, src, coerced);
7562 return val.toEnum(std.builtin.AtomicRmwOp);
7563}
7564
75527565fn zirCmpxchg(
75537566 sema: *Sema,
75547567 block: *Scope.Block,
......@@ -7664,14 +7677,108 @@ fn zirSelect(sema: *Sema, block: *Scope.Block, inst: Zir.Inst.Index) CompileErro
76647677
76657678fn zirAtomicLoad(sema: *Sema, block: *Scope.Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
76667679 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
7667 const src = inst_data.src();
7668 return sema.mod.fail(&block.base, src, "TODO: Sema.zirAtomicLoad", .{});
7680 const extra = sema.code.extraData(Zir.Inst.Bin, inst_data.payload_index).data;
7681 // zig fmt: off
7682 const elem_ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
7683 const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
7684 const order_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };
7685 // zig fmt: on
7686 const ptr = sema.resolveInst(extra.lhs);
7687 const elem_ty = sema.typeOf(ptr).elemType();
7688 try sema.checkAtomicOperandType(block, elem_ty_src, elem_ty);
7689 const order = try sema.resolveAtomicOrder(block, order_src, extra.rhs);
7690
7691 switch (order) {
7692 .Release, .AcqRel => {
7693 return sema.mod.fail(
7694 &block.base,
7695 order_src,
7696 "@atomicLoad atomic ordering must not be Release or AcqRel",
7697 .{},
7698 );
7699 },
7700 else => {},
7701 }
7702
7703 if (try sema.resolveDefinedValue(block, ptr_src, ptr)) |ptr_val| {
7704 if (try ptr_val.pointerDeref(sema.arena)) |elem_val| {
7705 return sema.addConstant(elem_ty, elem_val);
7706 }
7707 }
7708
7709 try sema.requireRuntimeBlock(block, ptr_src);
7710 return block.addInst(.{
7711 .tag = .atomic_load,
7712 .data = .{ .atomic_load = .{
7713 .ptr = ptr,
7714 .order = order,
7715 } },
7716 });
76697717}
76707718
76717719fn zirAtomicRmw(sema: *Sema, block: *Scope.Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
7720 const mod = sema.mod;
76727721 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
7722 const extra = sema.code.extraData(Zir.Inst.AtomicRmw, inst_data.payload_index).data;
76737723 const src = inst_data.src();
7674 return sema.mod.fail(&block.base, src, "TODO: Sema.zirAtomicRmw", .{});
7724 // zig fmt: off
7725 const operand_ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
7726 const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
7727 const op_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };
7728 const operand_src : LazySrcLoc = .{ .node_offset_builtin_call_arg3 = inst_data.src_node };
7729 const order_src : LazySrcLoc = .{ .node_offset_builtin_call_arg4 = inst_data.src_node };
7730 // zig fmt: on
7731 const ptr = sema.resolveInst(extra.ptr);
7732 const operand_ty = sema.typeOf(ptr).elemType();
7733 try sema.checkAtomicOperandType(block, operand_ty_src, operand_ty);
7734 const op = try sema.resolveAtomicRmwOp(block, op_src, extra.operation);
7735
7736 switch (operand_ty.zigTypeTag()) {
7737 .Enum => if (op != .Xchg) {
7738 return mod.fail(&block.base, op_src, "@atomicRmw with enum only allowed with .Xchg", .{});
7739 },
7740 .Bool => if (op != .Xchg) {
7741 return mod.fail(&block.base, op_src, "@atomicRmw with bool only allowed with .Xchg", .{});
7742 },
7743 .Float => switch (op) {
7744 .Xchg, .Add, .Sub => {},
7745 else => return mod.fail(&block.base, op_src, "@atomicRmw with float only allowed with .Xchg, .Add, and .Sub", .{}),
7746 },
7747 else => {},
7748 }
7749 const operand = try sema.coerce(block, operand_ty, sema.resolveInst(extra.operand), operand_src);
7750 const order = try sema.resolveAtomicOrder(block, order_src, extra.ordering);
7751
7752 if (order == .Unordered) {
7753 return mod.fail(&block.base, order_src, "@atomicRmw atomic ordering must not be Unordered", .{});
7754 }
7755
7756 // special case zero bit types
7757 if (try sema.typeHasOnePossibleValue(block, operand_ty_src, operand_ty)) |val| {
7758 return sema.addConstant(operand_ty, val);
7759 }
7760
7761 const runtime_src = if (try sema.resolveDefinedValue(block, ptr_src, ptr)) |ptr_val| rs: {
7762 if (try sema.resolveMaybeUndefVal(block, operand_src, operand)) |operand_val| {
7763 _ = ptr_val;
7764 _ = operand_val;
7765 return mod.fail(&block.base, src, "TODO implement Sema for @atomicRmw at comptime", .{});
7766 } else break :rs operand_src;
7767 } else ptr_src;
7768
7769 const flags: u32 = @as(u32, @enumToInt(order)) | (@as(u32, @enumToInt(op)) << 3);
7770
7771 try sema.requireRuntimeBlock(block, runtime_src);
7772 return block.addInst(.{
7773 .tag = .atomic_rmw,
7774 .data = .{ .pl_op = .{
7775 .operand = ptr,
7776 .payload = try sema.addExtra(Air.AtomicRmw{
7777 .operand = operand,
7778 .flags = flags,
7779 }),
7780 } },
7781 });
76757782}
76767783
76777784fn zirAtomicStore(sema: *Sema, block: *Scope.Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
......@@ -8848,7 +8955,7 @@ fn coerce(
88488955 if (dest_type.eql(inst_ty))
88498956 return inst;
88508957
8851 const in_memory_result = coerceInMemoryAllowed(dest_type, inst_ty);
8958 const in_memory_result = coerceInMemoryAllowed(dest_type, inst_ty, false);
88528959 if (in_memory_result == .ok) {
88538960 return sema.bitcast(block, dest_type, inst, inst_src);
88548961 }
......@@ -8890,11 +8997,12 @@ fn coerce(
88908997 const array_type = inst_ty.elemType();
88918998 if (array_type.zigTypeTag() != .Array) break :src_array_ptr;
88928999 const array_elem_type = array_type.elemType();
8893 if (inst_ty.isConstPtr() and !dest_type.isConstPtr()) break :src_array_ptr;
9000 const dest_is_mut = !dest_type.isConstPtr();
9001 if (inst_ty.isConstPtr() and dest_is_mut) break :src_array_ptr;
88949002 if (inst_ty.isVolatilePtr() and !dest_type.isVolatilePtr()) break :src_array_ptr;
88959003
88969004 const dst_elem_type = dest_type.elemType();
8897 switch (coerceInMemoryAllowed(dst_elem_type, array_elem_type)) {
9005 switch (coerceInMemoryAllowed(dst_elem_type, array_elem_type, dest_is_mut)) {
88989006 .ok => {},
88999007 .no_match => break :src_array_ptr,
89009008 }
......@@ -9001,10 +9109,80 @@ const InMemoryCoercionResult = enum {
90019109 no_match,
90029110};
90039111
9004fn coerceInMemoryAllowed(dest_type: Type, src_type: Type) InMemoryCoercionResult {
9112/// If pointers have the same representation in runtime memory, a bitcast AIR instruction
9113/// may be used for the coercion.
9114/// * `const` attribute can be gained
9115/// * `volatile` attribute can be gained
9116/// * `allowzero` attribute can be gained (whether from explicit attribute, C pointer, or optional pointer) but only if !dest_is_mut
9117/// * alignment can be decreased
9118/// * bit offset attributes must match exactly
9119/// * `*`/`[*]` must match exactly, but `[*c]` matches either one
9120/// * sentinel-terminated pointers can coerce into `[*]`
9121/// TODO improve this function to report recursive compile errors like it does in stage1.
9122/// look at the function types_match_const_cast_only
9123fn coerceInMemoryAllowed(dest_type: Type, src_type: Type, dest_is_mut: bool) InMemoryCoercionResult {
90059124 if (dest_type.eql(src_type))
90069125 return .ok;
90079126
9127 if (dest_type.zigTypeTag() == .Pointer and
9128 src_type.zigTypeTag() == .Pointer)
9129 {
9130 const dest_info = dest_type.ptrInfo().data;
9131 const src_info = src_type.ptrInfo().data;
9132
9133 const child = coerceInMemoryAllowed(dest_info.pointee_type, src_info.pointee_type, dest_info.mutable);
9134 if (child == .no_match) {
9135 return child;
9136 }
9137
9138 const ok_sent = dest_info.sentinel == null or src_info.size == .C or
9139 (src_info.sentinel != null and
9140 dest_info.sentinel.?.eql(src_info.sentinel.?, dest_info.pointee_type));
9141 if (!ok_sent) {
9142 return .no_match;
9143 }
9144
9145 const ok_ptr_size = src_info.size == dest_info.size or
9146 src_info.size == .C or dest_info.size == .C;
9147 if (!ok_ptr_size) {
9148 return .no_match;
9149 }
9150
9151 const ok_cv_qualifiers =
9152 (src_info.mutable or !dest_info.mutable) and
9153 (!src_info.@"volatile" or dest_info.@"volatile");
9154
9155 if (!ok_cv_qualifiers) {
9156 return .no_match;
9157 }
9158
9159 const ok_allows_zero = (dest_info.@"allowzero" and
9160 (src_info.@"allowzero" or !dest_is_mut)) or
9161 (!dest_info.@"allowzero" and !src_info.@"allowzero");
9162 if (!ok_allows_zero) {
9163 return .no_match;
9164 }
9165
9166 if (dest_type.hasCodeGenBits() != src_type.hasCodeGenBits()) {
9167 return .no_match;
9168 }
9169
9170 if (src_info.host_size != dest_info.host_size or
9171 src_info.bit_offset != dest_info.bit_offset)
9172 {
9173 return .no_match;
9174 }
9175
9176 assert(src_info.@"align" != 0);
9177 assert(dest_info.@"align" != 0);
9178
9179 if (dest_info.@"align" > src_info.@"align") {
9180 return .no_match;
9181 }
9182
9183 return .ok;
9184 }
9185
90089186 // TODO: implement more of this function
90099187
90109188 return .no_match;
src/codegen.zig+23
......@@ -860,6 +860,13 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
860860 .array_to_slice => try self.airArrayToSlice(inst),
861861 .cmpxchg_strong => try self.airCmpxchg(inst),
862862 .cmpxchg_weak => try self.airCmpxchg(inst),
863 .atomic_rmw => try self.airAtomicRmw(inst),
864 .atomic_load => try self.airAtomicLoad(inst),
865
866 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
867 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
868 .atomic_store_release => try self.airAtomicStore(inst, .Release),
869 .atomic_store_seq_cst => try self.airAtomicStore(inst, .SeqCst),
863870
864871 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
865872 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
......@@ -4773,6 +4780,22 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
47734780 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
47744781 }
47754782
4783 fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
4784 _ = inst;
4785 return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
4786 }
4787
4788 fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
4789 _ = inst;
4790 return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
4791 }
4792
4793 fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
4794 _ = inst;
4795 _ = order;
4796 return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
4797 }
4798
47764799 fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
47774800 // First section of indexes correspond to a set number of constant values.
47784801 const ref_int = @enumToInt(inst);
src/codegen/c.zig+80-3
......@@ -914,6 +914,13 @@ fn genBody(o: *Object, body: []const Air.Inst.Index) error{ AnalysisFail, OutOfM
914914 .array_to_slice => try airArrayToSlice(o, inst),
915915 .cmpxchg_weak => try airCmpxchg(o, inst, "weak"),
916916 .cmpxchg_strong => try airCmpxchg(o, inst, "strong"),
917 .atomic_rmw => try airAtomicRmw(o, inst),
918 .atomic_load => try airAtomicLoad(o, inst),
919
920 .atomic_store_unordered => try airAtomicStore(o, inst, toMemoryOrder(.Unordered)),
921 .atomic_store_monotonic => try airAtomicStore(o, inst, toMemoryOrder(.Monotonic)),
922 .atomic_store_release => try airAtomicStore(o, inst, toMemoryOrder(.Release)),
923 .atomic_store_seq_cst => try airAtomicStore(o, inst, toMemoryOrder(.SeqCst)),
917924
918925 .struct_field_ptr_index_0 => try airStructFieldPtrIndex(o, inst, 0),
919926 .struct_field_ptr_index_1 => try airStructFieldPtrIndex(o, inst, 1),
......@@ -1917,8 +1924,61 @@ fn airCmpxchg(o: *Object, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue {
19171924 return local;
19181925}
19191926
1920fn writeMemoryOrder(w: anytype, order: std.builtin.AtomicOrder) !void {
1921 const str = switch (order) {
1927fn airAtomicRmw(o: *Object, inst: Air.Inst.Index) !CValue {
1928 const pl_op = o.air.instructions.items(.data)[inst].pl_op;
1929 const extra = o.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1930 const inst_ty = o.air.typeOfIndex(inst);
1931 const ptr = try o.resolveInst(pl_op.operand);
1932 const operand = try o.resolveInst(extra.operand);
1933 const local = try o.allocLocal(inst_ty, .Const);
1934 const writer = o.writer();
1935
1936 try writer.print(" = zig_atomicrmw_{s}(", .{toAtomicRmwSuffix(extra.op())});
1937 try o.writeCValue(writer, ptr);
1938 try writer.writeAll(", ");
1939 try o.writeCValue(writer, operand);
1940 try writer.writeAll(", ");
1941 try writeMemoryOrder(writer, extra.ordering());
1942 try writer.writeAll(");\n");
1943
1944 return local;
1945}
1946
1947fn airAtomicLoad(o: *Object, inst: Air.Inst.Index) !CValue {
1948 const atomic_load = o.air.instructions.items(.data)[inst].atomic_load;
1949 const inst_ty = o.air.typeOfIndex(inst);
1950 const ptr = try o.resolveInst(atomic_load.ptr);
1951 const local = try o.allocLocal(inst_ty, .Const);
1952 const writer = o.writer();
1953
1954 try writer.writeAll(" = zig_atomic_load(");
1955 try o.writeCValue(writer, ptr);
1956 try writer.writeAll(", ");
1957 try writeMemoryOrder(writer, atomic_load.order);
1958 try writer.writeAll(");\n");
1959
1960 return local;
1961}
1962
1963fn airAtomicStore(o: *Object, inst: Air.Inst.Index, order: [*:0]const u8) !CValue {
1964 const bin_op = o.air.instructions.items(.data)[inst].bin_op;
1965 const ptr = try o.resolveInst(bin_op.lhs);
1966 const element = try o.resolveInst(bin_op.rhs);
1967 const inst_ty = o.air.typeOfIndex(inst);
1968 const local = try o.allocLocal(inst_ty, .Const);
1969 const writer = o.writer();
1970
1971 try writer.writeAll(" = zig_atomic_store(");
1972 try o.writeCValue(writer, ptr);
1973 try writer.writeAll(", ");
1974 try o.writeCValue(writer, element);
1975 try writer.print(", {s});\n", .{order});
1976
1977 return local;
1978}
1979
1980fn toMemoryOrder(order: std.builtin.AtomicOrder) [:0]const u8 {
1981 return switch (order) {
19221982 .Unordered => "memory_order_relaxed",
19231983 .Monotonic => "memory_order_consume",
19241984 .Acquire => "memory_order_acquire",
......@@ -1926,7 +1986,24 @@ fn writeMemoryOrder(w: anytype, order: std.builtin.AtomicOrder) !void {
19261986 .AcqRel => "memory_order_acq_rel",
19271987 .SeqCst => "memory_order_seq_cst",
19281988 };
1929 return w.writeAll(str);
1989}
1990
1991fn writeMemoryOrder(w: anytype, order: std.builtin.AtomicOrder) !void {
1992 return w.writeAll(toMemoryOrder(order));
1993}
1994
1995fn toAtomicRmwSuffix(order: std.builtin.AtomicRmwOp) []const u8 {
1996 return switch (order) {
1997 .Xchg => "xchg",
1998 .Add => "add",
1999 .Sub => "sub",
2000 .And => "and",
2001 .Nand => "nand",
2002 .Or => "or",
2003 .Xor => "xor",
2004 .Max => "max",
2005 .Min => "min",
2006 };
19302007}
19312008
19322009fn IndentWriter(comptime UnderlyingWriter: type) type {
src/codegen/llvm.zig+150-10
......@@ -1060,6 +1060,13 @@ pub const FuncGen = struct {
10601060 .cmpxchg_weak => try self.airCmpxchg(inst, true),
10611061 .cmpxchg_strong => try self.airCmpxchg(inst, false),
10621062 .fence => try self.airFence(inst),
1063 .atomic_rmw => try self.airAtomicRmw(inst),
1064 .atomic_load => try self.airAtomicLoad(inst),
1065
1066 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
1067 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
1068 .atomic_store_release => try self.airAtomicStore(inst, .Release),
1069 .atomic_store_seq_cst => try self.airAtomicStore(inst, .SequentiallyConsistent),
10631070
10641071 .struct_field_ptr => try self.airStructFieldPtr(inst),
10651072 .struct_field_val => try self.airStructFieldVal(inst),
......@@ -1983,20 +1990,19 @@ pub const FuncGen = struct {
19831990 fn airStore(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
19841991 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
19851992 const dest_ptr = try self.resolveInst(bin_op.lhs);
1993 const ptr_ty = self.air.typeOf(bin_op.lhs);
19861994 const src_operand = try self.resolveInst(bin_op.rhs);
1987 // TODO set volatile on this store properly
1988 _ = self.builder.buildStore(src_operand, dest_ptr);
1995 _ = self.store(dest_ptr, ptr_ty, src_operand);
19891996 return null;
19901997 }
19911998
19921999 fn airLoad(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
19932000 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1994 const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr();
1995 if (!is_volatile and self.liveness.isUnused(inst))
2001 const ptr_ty = self.air.typeOf(ty_op.operand);
2002 if (!ptr_ty.isVolatilePtr() and self.liveness.isUnused(inst))
19962003 return null;
19972004 const ptr = try self.resolveInst(ty_op.operand);
1998 // TODO set volatile on this load properly
1999 return self.builder.buildLoad(ptr, "");
2005 return self.load(ptr, ptr_ty);
20002006 }
20012007
20022008 fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
......@@ -2033,14 +2039,12 @@ pub const FuncGen = struct {
20332039 new_value = self.builder.buildZExt(new_value, abi_ty, "");
20342040 }
20352041 }
2036 const success_order = toLlvmAtomicOrdering(extra.successOrder());
2037 const failure_order = toLlvmAtomicOrdering(extra.failureOrder());
20382042 const result = self.builder.buildCmpXchg(
20392043 ptr,
20402044 expected_value,
20412045 new_value,
2042 success_order,
2043 failure_order,
2046 toLlvmAtomicOrdering(extra.successOrder()),
2047 toLlvmAtomicOrdering(extra.failureOrder()),
20442048 is_weak,
20452049 self.single_threaded,
20462050 );
......@@ -2066,6 +2070,109 @@ pub const FuncGen = struct {
20662070 return self.builder.buildInsertValue(partial, non_null_bit, 1, "");
20672071 }
20682072
2073 fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
2074 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2075 const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
2076 const ptr = try self.resolveInst(pl_op.operand);
2077 const ptr_ty = self.air.typeOf(pl_op.operand);
2078 const operand_ty = ptr_ty.elemType();
2079 const operand = try self.resolveInst(extra.operand);
2080 const is_signed_int = operand_ty.isSignedInt();
2081 const is_float = operand_ty.isFloat();
2082 const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
2083 const ordering = toLlvmAtomicOrdering(extra.ordering());
2084 const single_threaded = llvm.Bool.fromBool(self.single_threaded);
2085 const opt_abi_ty = self.dg.getAtomicAbiType(operand_ty, op == .Xchg);
2086 if (opt_abi_ty) |abi_ty| {
2087 // operand needs widening and truncating or bitcasting.
2088 const casted_ptr = self.builder.buildBitCast(ptr, abi_ty.pointerType(0), "");
2089 const casted_operand = if (is_float)
2090 self.builder.buildBitCast(operand, abi_ty, "")
2091 else if (is_signed_int)
2092 self.builder.buildSExt(operand, abi_ty, "")
2093 else
2094 self.builder.buildZExt(operand, abi_ty, "");
2095
2096 const uncasted_result = self.builder.buildAtomicRmw(
2097 op,
2098 casted_ptr,
2099 casted_operand,
2100 ordering,
2101 single_threaded,
2102 );
2103 const operand_llvm_ty = try self.dg.llvmType(operand_ty);
2104 if (is_float) {
2105 return self.builder.buildBitCast(uncasted_result, operand_llvm_ty, "");
2106 } else {
2107 return self.builder.buildTrunc(uncasted_result, operand_llvm_ty, "");
2108 }
2109 }
2110
2111 if (operand.typeOf().getTypeKind() != .Pointer) {
2112 return self.builder.buildAtomicRmw(op, ptr, operand, ordering, single_threaded);
2113 }
2114
2115 // It's a pointer but we need to treat it as an int.
2116 const usize_llvm_ty = try self.dg.llvmType(Type.initTag(.usize));
2117 const casted_ptr = self.builder.buildBitCast(ptr, usize_llvm_ty.pointerType(0), "");
2118 const casted_operand = self.builder.buildPtrToInt(operand, usize_llvm_ty, "");
2119 const uncasted_result = self.builder.buildAtomicRmw(
2120 op,
2121 casted_ptr,
2122 casted_operand,
2123 ordering,
2124 single_threaded,
2125 );
2126 const operand_llvm_ty = try self.dg.llvmType(operand_ty);
2127 return self.builder.buildIntToPtr(uncasted_result, operand_llvm_ty, "");
2128 }
2129
2130 fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
2131 const atomic_load = self.air.instructions.items(.data)[inst].atomic_load;
2132 const ptr = try self.resolveInst(atomic_load.ptr);
2133 const ptr_ty = self.air.typeOf(atomic_load.ptr);
2134 const ordering = toLlvmAtomicOrdering(atomic_load.order);
2135 const operand_ty = ptr_ty.elemType();
2136 const opt_abi_ty = self.dg.getAtomicAbiType(operand_ty, false);
2137
2138 if (opt_abi_ty) |abi_ty| {
2139 // operand needs widening and truncating
2140 const casted_ptr = self.builder.buildBitCast(ptr, abi_ty.pointerType(0), "");
2141 const load_inst = self.load(casted_ptr, ptr_ty);
2142 load_inst.setOrdering(ordering);
2143 return self.builder.buildTrunc(load_inst, try self.dg.llvmType(operand_ty), "");
2144 }
2145 const load_inst = self.load(ptr, ptr_ty);
2146 load_inst.setOrdering(ordering);
2147 return load_inst;
2148 }
2149
2150 fn airAtomicStore(
2151 self: *FuncGen,
2152 inst: Air.Inst.Index,
2153 ordering: llvm.AtomicOrdering,
2154 ) !?*const llvm.Value {
2155 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2156 var ptr = try self.resolveInst(bin_op.lhs);
2157 const ptr_ty = self.air.typeOf(bin_op.lhs);
2158 var element = try self.resolveInst(bin_op.rhs);
2159 const operand_ty = ptr_ty.elemType();
2160 const opt_abi_ty = self.dg.getAtomicAbiType(operand_ty, false);
2161
2162 if (opt_abi_ty) |abi_ty| {
2163 // operand needs widening
2164 ptr = self.builder.buildBitCast(ptr, abi_ty.pointerType(0), "");
2165 if (operand_ty.isSignedInt()) {
2166 element = self.builder.buildSExt(element, abi_ty, "");
2167 } else {
2168 element = self.builder.buildZExt(element, abi_ty, "");
2169 }
2170 }
2171 const store_inst = self.store(ptr, ptr_ty, element);
2172 store_inst.setOrdering(ordering);
2173 return null;
2174 }
2175
20692176 fn getIntrinsic(self: *FuncGen, name: []const u8) *const llvm.Value {
20702177 const id = llvm.lookupIntrinsicID(name.ptr, name.len);
20712178 assert(id != 0);
......@@ -2074,6 +2181,21 @@ pub const FuncGen = struct {
20742181 // `getIntrinsicDeclaration`
20752182 return self.llvmModule().getIntrinsicDeclaration(id, null, 0);
20762183 }
2184
2185 fn load(self: *FuncGen, ptr: *const llvm.Value, ptr_ty: Type) *const llvm.Value {
2186 _ = ptr_ty; // TODO set volatile and alignment on this load properly
2187 return self.builder.buildLoad(ptr, "");
2188 }
2189
2190 fn store(
2191 self: *FuncGen,
2192 ptr: *const llvm.Value,
2193 ptr_ty: Type,
2194 elem: *const llvm.Value,
2195 ) *const llvm.Value {
2196 _ = ptr_ty; // TODO set volatile and alignment on this store properly
2197 return self.builder.buildStore(elem, ptr);
2198 }
20772199};
20782200
20792201fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {
......@@ -2227,3 +2349,21 @@ fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) llvm.AtomicOrderi
22272349 .SeqCst => .SequentiallyConsistent,
22282350 };
22292351}
2352
2353fn toLlvmAtomicRmwBinOp(
2354 op: std.builtin.AtomicRmwOp,
2355 is_signed: bool,
2356 is_float: bool,
2357) llvm.AtomicRMWBinOp {
2358 return switch (op) {
2359 .Xchg => .Xchg,
2360 .Add => if (is_float) llvm.AtomicRMWBinOp.FAdd else return .Add,
2361 .Sub => if (is_float) llvm.AtomicRMWBinOp.FSub else return .Sub,
2362 .And => .And,
2363 .Nand => .Nand,
2364 .Or => .Or,
2365 .Xor => .Xor,
2366 .Max => if (is_signed) llvm.AtomicRMWBinOp.Max else return .UMax,
2367 .Min => if (is_signed) llvm.AtomicRMWBinOp.Min else return .UMin,
2368 };
2369}
src/codegen/llvm/bindings.zig+63
......@@ -133,6 +133,9 @@ pub const Value = opaque {
133133
134134 pub const constIntToPtr = LLVMConstIntToPtr;
135135 extern fn LLVMConstIntToPtr(ConstantVal: *const Value, ToType: *const Type) *const Value;
136
137 pub const setOrdering = LLVMSetOrdering;
138 extern fn LLVMSetOrdering(MemoryAccessInst: *const Value, Ordering: AtomicOrdering) void;
136139};
137140
138141pub const Type = opaque {
......@@ -167,6 +170,9 @@ pub const Type = opaque {
167170 ElementCount: c_uint,
168171 Packed: Bool,
169172 ) void;
173
174 pub const getTypeKind = LLVMGetTypeKind;
175 extern fn LLVMGetTypeKind(Ty: *const Type) TypeKind;
170176};
171177
172178pub const Module = opaque {
......@@ -477,6 +483,14 @@ pub const Builder = opaque {
477483 Name: [*:0]const u8,
478484 ) *const Value;
479485
486 pub const buildIntToPtr = LLVMBuildIntToPtr;
487 extern fn LLVMBuildIntToPtr(
488 *const Builder,
489 Val: *const Value,
490 DestTy: *const Type,
491 Name: [*:0]const u8,
492 ) *const Value;
493
480494 pub const buildStructGEP = LLVMBuildStructGEP;
481495 extern fn LLVMBuildStructGEP(
482496 B: *const Builder,
......@@ -530,6 +544,16 @@ pub const Builder = opaque {
530544 singleThread: Bool,
531545 Name: [*:0]const u8,
532546 ) *const Value;
547
548 pub const buildAtomicRmw = LLVMBuildAtomicRMW;
549 extern fn LLVMBuildAtomicRMW(
550 B: *const Builder,
551 op: AtomicRMWBinOp,
552 PTR: *const Value,
553 Val: *const Value,
554 ordering: AtomicOrdering,
555 singleThread: Bool,
556 ) *const Value;
533557};
534558
535559pub const IntPredicate = enum(c_uint) {
......@@ -901,3 +925,42 @@ pub const AtomicOrdering = enum(c_uint) {
901925 AcquireRelease = 6,
902926 SequentiallyConsistent = 7,
903927};
928
929pub const AtomicRMWBinOp = enum(c_int) {
930 Xchg,
931 Add,
932 Sub,
933 And,
934 Nand,
935 Or,
936 Xor,
937 Max,
938 Min,
939 UMax,
940 UMin,
941 FAdd,
942 FSub,
943};
944
945pub const TypeKind = enum(c_int) {
946 Void,
947 Half,
948 Float,
949 Double,
950 X86_FP80,
951 FP128,
952 PPC_FP128,
953 Label,
954 Integer,
955 Function,
956 Struct,
957 Array,
958 Pointer,
959 Vector,
960 Metadata,
961 X86_MMX,
962 Token,
963 ScalableVector,
964 BFloat,
965 X86_AMX,
966};
src/link/C/zig.h+51-6
......@@ -62,16 +62,61 @@
6262
6363#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
6464#include <stdatomic.h>
65#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail)
66#define zig_cmpxchg_weak(obj, expected, desired, succ, fail) atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail)
65#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)
66#define zig_cmpxchg_weak (obj, expected, desired, succ, fail) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail)
67#define zig_atomicrmw_xchg(obj, arg, order) atomic_exchange_explicit (obj, arg, order)
68#define zig_atomicrmw_add (obj, arg, order) atomic_fetch_add_explicit (obj, arg, order)
69#define zig_atomicrmw_sub (obj, arg, order) atomic_fetch_sub_explicit (obj, arg, order)
70#define zig_atomicrmw_or (obj, arg, order) atomic_fetch_or_explicit (obj, arg, order)
71#define zig_atomicrmw_xor (obj, arg, order) atomic_fetch_xor_explicit (obj, arg, order)
72#define zig_atomicrmw_and (obj, arg, order) atomic_fetch_and_explicit (obj, arg, order)
73#define zig_atomicrmw_nand(obj, arg, order) atomic_fetch_nand_explicit(obj, arg, order)
74#define zig_atomicrmw_min (obj, arg, order) atomic_fetch_min_explicit (obj, arg, order)
75#define zig_atomicrmw_max (obj, arg, order) atomic_fetch_max_explicit (obj, arg, order)
76#define zig_atomic_store (obj, arg, order) atomic_store_explicit (obj, arg, order)
77#define zig_atomic_load (obj, order) atomic_load_explicit (obj, order)
6778#define zig_fence(order) atomic_thread_fence(order)
6879#elif __GNUC__
69#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) __sync_val_compare_and_swap(obj, expected, desired)
70#define zig_cmpxchg_weak(obj, expected, desired, succ, fail) __sync_val_compare_and_swap(obj, expected, desired)
71#define zig_fence(order) __sync_synchronize(order)
80#define memory_order_relaxed __ATOMIC_RELAXED
81#define memory_order_consume __ATOMIC_CONSUME
82#define memory_order_acquire __ATOMIC_ACQUIRE
83#define memory_order_release __ATOMIC_RELEASE
84#define memory_order_acq_rel __ATOMIC_ACQ_REL
85#define memory_order_seq_cst __ATOMIC_SEQ_CST
86#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) __atomic_compare_exchange_n(obj, &(expected), desired, false, succ, fail)
87#define zig_cmpxchg_weak (obj, expected, desired, succ, fail) __atomic_compare_exchange_n(obj, &(expected), desired, true , succ, fail)
88#define zig_atomicrmw_xchg(obj, arg, order) __atomic_exchange_n(obj, arg, order)
89#define zig_atomicrmw_add (obj, arg, order) __atomic_fetch_add (obj, arg, order)
90#define zig_atomicrmw_sub (obj, arg, order) __atomic_fetch_sub (obj, arg, order)
91#define zig_atomicrmw_or (obj, arg, order) __atomic_fetch_or (obj, arg, order)
92#define zig_atomicrmw_xor (obj, arg, order) __atomic_fetch_xor (obj, arg, order)
93#define zig_atomicrmw_and (obj, arg, order) __atomic_fetch_and (obj, arg, order)
94#define zig_atomicrmw_nand(obj, arg, order) __atomic_fetch_nand(obj, arg, order)
95#define zig_atomicrmw_min (obj, arg, order) __atomic_fetch_min (obj, arg, order)
96#define zig_atomicrmw_max (obj, arg, order) __atomic_fetch_max (obj, arg, order)
97#define zig_atomic_store (obj, arg, order) __atomic_store (obj, arg, order)
98#define zig_atomic_load (obj, order) __atomic_load (obj, order)
99#define zig_fence(order) __atomic_thread_fence(order)
72100#else
101#define memory_order_relaxed 0
102#define memory_order_consume 1
103#define memory_order_acquire 2
104#define memory_order_release 3
105#define memory_order_acq_rel 4
106#define memory_order_seq_cst 5
73107#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) zig_unimplemented()
74#define zig_cmpxchg_weak(obj, expected, desired, succ, fail) zig_unimplemented()
108#define zig_cmpxchg_weak (obj, expected, desired, succ, fail) zig_unimplemented()
109#define zig_atomicrmw_xchg(obj, arg, order) zig_unimplemented()
110#define zig_atomicrmw_add (obj, arg, order) zig_unimplemented()
111#define zig_atomicrmw_sub (obj, arg, order) zig_unimplemented()
112#define zig_atomicrmw_or (obj, arg, order) zig_unimplemented()
113#define zig_atomicrmw_xor (obj, arg, order) zig_unimplemented()
114#define zig_atomicrmw_and (obj, arg, order) zig_unimplemented()
115#define zig_atomicrmw_nand(obj, arg, order) zig_unimplemented()
116#define zig_atomicrmw_min (obj, arg, order) zig_unimplemented()
117#define zig_atomicrmw_max (obj, arg, order) zig_unimplemented()
118#define zig_atomic_store (obj, arg, order) zig_unimplemented()
119#define zig_atomic_load (obj, order) zig_unimplemented()
75120#define zig_fence(order) zig_unimplemented()
76121#endif
77122
src/print_air.zig+36
......@@ -193,6 +193,12 @@ const Writer = struct {
193193 .switch_br => try w.writeSwitchBr(s, inst),
194194 .cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),
195195 .fence => try w.writeFence(s, inst),
196 .atomic_load => try w.writeAtomicLoad(s, inst),
197 .atomic_store_unordered => try w.writeAtomicStore(s, inst, .Unordered),
198 .atomic_store_monotonic => try w.writeAtomicStore(s, inst, .Monotonic),
199 .atomic_store_release => try w.writeAtomicStore(s, inst, .Release),
200 .atomic_store_seq_cst => try w.writeAtomicStore(s, inst, .SeqCst),
201 .atomic_rmw => try w.writeAtomicRmw(s, inst),
196202 }
197203 }
198204
......@@ -283,6 +289,36 @@ const Writer = struct {
283289 try s.print("{s}", .{@tagName(atomic_order)});
284290 }
285291
292 fn writeAtomicLoad(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
293 const atomic_load = w.air.instructions.items(.data)[inst].atomic_load;
294
295 try w.writeOperand(s, inst, 0, atomic_load.ptr);
296 try s.print(", {s}", .{@tagName(atomic_load.order)});
297 }
298
299 fn writeAtomicStore(
300 w: *Writer,
301 s: anytype,
302 inst: Air.Inst.Index,
303 order: std.builtin.AtomicOrder,
304 ) @TypeOf(s).Error!void {
305 const bin_op = w.air.instructions.items(.data)[inst].bin_op;
306 try w.writeOperand(s, inst, 0, bin_op.lhs);
307 try s.writeAll(", ");
308 try w.writeOperand(s, inst, 1, bin_op.rhs);
309 try s.print(", {s}", .{@tagName(order)});
310 }
311
312 fn writeAtomicRmw(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
313 const pl_op = w.air.instructions.items(.data)[inst].pl_op;
314 const extra = w.air.extraData(Air.AtomicRmw, pl_op.payload).data;
315
316 try w.writeOperand(s, inst, 0, pl_op.operand);
317 try s.writeAll(", ");
318 try w.writeOperand(s, inst, 1, extra.operand);
319 try s.print(", {s}, {s}", .{ @tagName(extra.op()), @tagName(extra.ordering()) });
320 }
321
286322 fn writeConstant(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
287323 const ty_pl = w.air.instructions.items(.data)[inst].ty_pl;
288324 const val = w.air.values[ty_pl.payload];
test/behavior/atomics.zig+23
......@@ -30,3 +30,26 @@ test "fence" {
3030 @fence(.SeqCst);
3131 x = 5678;
3232}
33
34test "atomicrmw and atomicload" {
35 var data: u8 = 200;
36 try testAtomicRmw(&data);
37 try expect(data == 42);
38 try testAtomicLoad(&data);
39}
40
41fn testAtomicRmw(ptr: *u8) !void {
42 const prev_value = @atomicRmw(u8, ptr, .Xchg, 42, .SeqCst);
43 try expect(prev_value == 200);
44 comptime {
45 var x: i32 = 1234;
46 const y: i32 = 12345;
47 try expect(@atomicLoad(i32, &x, .SeqCst) == 1234);
48 try expect(@atomicLoad(i32, &y, .SeqCst) == 12345);
49 }
50}
51
52fn testAtomicLoad(ptr: *u8) !void {
53 const x = @atomicLoad(u8, ptr, .SeqCst);
54 try expect(x == 42);
55}
test/behavior/atomics_stage1.zig-23
......@@ -3,29 +3,6 @@ const expect = std.testing.expect;
33const expectEqual = std.testing.expectEqual;
44const builtin = @import("builtin");
55
6test "atomicrmw and atomicload" {
7 var data: u8 = 200;
8 try testAtomicRmw(&data);
9 try expect(data == 42);
10 try testAtomicLoad(&data);
11}
12
13fn testAtomicRmw(ptr: *u8) !void {
14 const prev_value = @atomicRmw(u8, ptr, .Xchg, 42, .SeqCst);
15 try expect(prev_value == 200);
16 comptime {
17 var x: i32 = 1234;
18 const y: i32 = 12345;
19 try expect(@atomicLoad(i32, &x, .SeqCst) == 1234);
20 try expect(@atomicLoad(i32, &y, .SeqCst) == 12345);
21 }
22}
23
24fn testAtomicLoad(ptr: *u8) !void {
25 const x = @atomicLoad(u8, ptr, .SeqCst);
26 try expect(x == 42);
27}
28
296test "cmpxchg with ptr" {
307 var data1: i32 = 1234;
318 var data2: i32 = 5678;